├── .github └── workflows │ └── main.yml ├── .gitignore ├── LICENSE ├── README.md ├── README_old.md ├── app ├── __init__.py ├── config.py ├── functions │ ├── __init__.py │ ├── base.py │ └── duckduck.py ├── libs │ ├── __init__.py │ ├── base_handler.py │ ├── chains copy.py │ ├── chains.py │ ├── context.py │ ├── provider_handler.py │ ├── tools_handler.py │ └── vision_handler.py ├── main.py ├── models.py ├── prompts.py ├── providers.py ├── reasoning │ ├── __init__.py │ ├── base.py │ └── rerank.py ├── routes │ ├── __init__.py │ ├── examples.py │ └── proxy.py └── utils.py ├── cookbook ├── ai_assistant_custome_tools.py ├── cinemax.json ├── function_call_force_schema.py ├── function_call_force_tool_choice.py ├── function_call_ollama.py ├── function_call_phidata.py ├── function_call_vision.py ├── function_call_with_schema.py ├── function_call_without_schema.py ├── functiona_call_groq_langchain.py └── resources.py ├── examples ├── example_1.py ├── example_2.py ├── example_3.py └── example_4.py ├── frontend ├── assets │ ├── README.md │ ├── markdown.css │ └── style.css └── pages │ ├── index.html │ └── index_old.html └── requirements.txt /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Auto PR and Merge on Push by Specific User 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | auto-pr-and-merge: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Check User 13 | id: check_user 14 | run: | 15 | echo "user_matched=${{ github.actor == 'unclecode' }}" 16 | echo "user_matched=${{ github.actor == 'unclecode' }}" >> $GITHUB_ENV 17 | 18 | - name: Create Pull Request 19 | if: env.user_matched == 'true' 20 | id: create_pull_request 21 | uses: actions/github-script@v5 22 | with: 23 | script: | 24 | const payload = { 25 | owner: context.repo.owner, 26 | repo: context.repo.repo, 27 | head: 'main', 28 | base: 'live', 29 | title: 'Auto PR from main to live', 30 | body: 'Automatically generated PR to keep live branch up-to-date', 31 | draft: false, 32 | }; 33 | 34 | // Create the pull request 35 | await github.rest.pulls.create(payload).then(pr => { 36 | core.setOutput('pr_number', pr.data.number); 37 | }).catch(err => core.setFailed(`Failed to create PR: ${err.message}`)); 38 | 39 | - name: Merge Pull Request 40 | if: env.user_matched == 'true' 41 | uses: actions/github-script@v5 42 | with: 43 | script: | 44 | const pr_number = ${{ steps.create_pull_request.outputs.pr_number }}; 45 | if (!pr_number) { 46 | core.setFailed('PR number is undefined, skipping merge.'); 47 | return; 48 | } 49 | 50 | const payload = { 51 | owner: context.repo.owner, 52 | repo: context.repo.repo, 53 | pull_number: parseInt(pr_number, 10), 54 | merge_method: 'merge', // Options: 'merge', 'squash', or 'rebase' 55 | }; 56 | 57 | // Attempt to merge the pull request 58 | await github.rest.pulls.merge(payload).then(response => { 59 | if (response.status !== 200) { 60 | core.setFailed('Failed to merge the pull request'); 61 | } 62 | }).catch(err => core.setFailed(`Failed to merge PR: ${err.message}`)); 63 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | 162 | 163 | app.log 164 | .vscode 165 | app/routes/proxy_all_in_one.py -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. 10 | 11 | "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. 12 | 13 | "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. 14 | 15 | "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. 16 | 17 | "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. 18 | 19 | "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. 20 | 21 | "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). 22 | 23 | "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. 24 | 25 | "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." 26 | 27 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 28 | 29 | 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 30 | 31 | 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 32 | 33 | 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: 34 | 35 | You must give any other recipients of the Work or Derivative Works a copy of this License; and 36 | You must cause any modified files to carry prominent notices stating that You changed the files; and 37 | You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and 38 | If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. 39 | You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 40 | 41 | 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 42 | 43 | 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 44 | 45 | 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 46 | 47 | 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 48 | 49 | 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. 50 | 51 | END OF TERMS AND CONDITIONS 52 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GroqCall.ai - Lightning-Fast LLM Function Calls 2 | 3 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1q3is7qynCsx4s7FBznCfTMnokbKWIv1F?usp=sharing) 4 | [![Version](https://img.shields.io/badge/version-0.0.5-blue.svg)](https://github.com/unclecode/groqcall) 5 | [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 6 | 7 | GroqCall is a proxy server that enables lightning-fast function calls for Groq's Language Processing Unit (LPU) and other AI providers. It simplifies the creation of AI assistants by offering a wide range of built-in functions hosted on the cloud. 8 | 9 | ## Quickstart 10 | 11 | ### Using the Pre-built Server 12 | 13 | To quickly start using GroqCall without running it locally, make requests to one of the following base URLs: 14 | 15 | - Cloud: `https://groqcall.ai/proxy/groq/v1` 16 | - Local: `http://localhost:8000` (if running the proxy server locally) 17 | 18 | ### Running the Proxy Locally 19 | 20 | 1. Clone the repository: 21 | ``` 22 | git clone https://github.com/unclecode/groqcall.git 23 | cd groqcall 24 | ``` 25 | 26 | 2. Create and activate a virtual environment: 27 | ``` 28 | python -m venv venv 29 | source venv/bin/activate 30 | ``` 31 | 32 | 3. Install dependencies: 33 | ``` 34 | pip install -r requirements.txt 35 | ``` 36 | 37 | 4. Run the FastAPI server: 38 | ``` 39 | ./venv/bin/uvicorn --app-dir app/ main:app --reload 40 | ``` 41 | 42 | ## Examples 43 | 44 | ### Using GroqCall with PhiData 45 | 46 | ```python 47 | from phi.llm.openai.like import OpenAILike 48 | from phi.assistant import Assistant 49 | from phi.tools.duckduckgo import DuckDuckGo 50 | 51 | my_groq = OpenAILike( 52 | model="mixtral-8x7b-32768", 53 | api_key="YOUR_GROQ_API_KEY", 54 | base_url="https://groqcall.ai/proxy/groq/v1" # or "http://localhost:8000/proxy/groq/v1" if running locally 55 | ) 56 | 57 | assistant = Assistant( 58 | llm=my_groq, 59 | tools=[DuckDuckGo()], 60 | show_tool_calls=True, 61 | markdown=True 62 | ) 63 | 64 | assistant.print_response("What's happening in France? Summarize top stories with sources, very short and concise.", stream=False) 65 | ``` 66 | 67 | ### Using GroqCall with Requests 68 | 69 | #### FuncHub: Schema-less Function Calls 70 | 71 | GroqCall introduces FuncHub, which allows you to make function calls without passing the function schema. 72 | 73 | ```python 74 | import requests 75 | 76 | api_key = "YOUR_GROQ_API_KEY" 77 | header = { 78 | "Authorization": f"Bearer {api_key}", 79 | "Content-Type": "application/json" 80 | } 81 | 82 | proxy_url = "https://groqcall.ai/proxy/groq/v1/chat/completions" # or "http://localhost:8000/proxy/groq/v1/chat/completions" if running locally 83 | 84 | request = { 85 | "messages": [ 86 | { 87 | "role": "system", 88 | "content": "YOU MUST FOLLOW THESE INSTRUCTIONS CAREFULLY.\n\n1. Use markdown to format your answers.\n" 89 | }, 90 | { 91 | "role": "user", 92 | "content": "What's happening in France? Summarize top stories with sources, very short and concise." 93 | } 94 | ], 95 | "model": "mixtral-8x7b-32768", 96 | "tool_choice": "auto", 97 | "tools": [ 98 | { 99 | "type": "function", 100 | "function": { 101 | "name": "duckduck.search" 102 | } 103 | }, 104 | { 105 | "type": "function", 106 | "function": { 107 | "name": "duckduck.news" 108 | } 109 | } 110 | ] 111 | } 112 | 113 | response = requests.post( 114 | proxy_url, 115 | headers=header, 116 | json=request 117 | ) 118 | 119 | print(response.json()["choices"][0]["message"]["content"]) 120 | ``` 121 | 122 | - If you notice, the function schema is not passed in the request. This is because GroqCall uses FuncHub to automatically detect and call the function based on the function name in the cloud, Therefore you dont't need to parse the first response, call the function, and pass again. Check "functions" folder to add your own functions. I will create more examples in the close future to explain how to add your own functions. 123 | 124 | #### Passing Function Schemas 125 | 126 | If you prefer to pass your own function schemas, refer to the [Function Schema example](https://github.com/unclecode/groqcall/blob/main/cookbook/function_call_with_schema.py) in the cookbook. 127 | 128 | #### Rune proxy with Ollama locally 129 | 130 | Function call proxy can be used with Ollama. You should first install Ollama and run it locally. Then refer to the [Ollama example](https://github.com/unclecode/groqcall/blob/main/cookbook/function_call_ollama.py) in the cookbook. 131 | 132 | ## Cookbook 133 | 134 | Explore the [Cookbook](https://github.com/unclecode/groqcall/tree/main/cookbook) for more examples and use cases of GroqCall. 135 | 136 | ## Motivation 137 | 138 | Groq is a startup that designs highly specialized processor chips aimed specifically at running inference on large language models. They've introduced what they call the Language Processing Unit (LPU), and the speed is astounding—capable of producing 500 to 800 tokens per second or more. 139 | 140 | As an admirer of Groq and their community, I built this proxy to enable function calls using the OpenAI interface, allowing it to be called from any library. This engineering workaround has proven to be immensely useful in my company for various projects. 141 | 142 | ## Contributing 143 | 144 | Contributions are welcome! If you have ideas, suggestions, or would like to contribute to this project, please reach out to me on Twitter (X) @unclecode or via email at unclecode@kidocode.com. 145 | 146 | Let's collaborate and make this repository even more awesome! 🚀 147 | 148 | ## License 149 | 150 | This project is licensed under the Apache License 2.0. See [LICENSE](https://github.com/unclecode/groqcall/blob/main/LICENSE) for more information. -------------------------------------------------------------------------------- /README_old.md: -------------------------------------------------------------------------------- 1 | # GroqCall.ai (I changed the name from FunckyCall to GroqCall) 2 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1q3is7qynCsx4s7FBznCfTMnokbKWIv1F?usp=sharing) 3 | [![Version](https://img.shields.io/badge/version-0.0.1-blue.svg)](https://github.com/unclecode/groqcall) 4 | [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 5 | 6 | GroqCall is a proxy server provides function call for Groq's lightning-fast Language Processing Unit (LPU) and other AI providers. Additionally, the upcoming FuncyHub will offer a wide range of built-in functions, hosted on the cloud, making it easier to create AI assistants without the need to maintain function schemas in the codebase or or execute them through multiple calls. 7 | 8 | ## Motivation 🚀 9 | Groq is a startup that designs highly specialized processor chips aimed specifically at running inference on large language models. They've introduced what they call the Language Processing Unit (LPU), and the speed is astounding—capable of producing 500 to 800 tokens per second or more. I've become a big fan of Groq and their community; 10 | 11 | 12 | I admire what they're doing. It feels like after discovering electricity, the next challenge is moving it around quickly and efficiently. Groq is doing just that for Artificial Intelligence, making it easily accessible everywhere. They've opened up their API to the cloud, but as of now, they lack a function call capability. 13 | 14 | Unable to wait for this feature, I built a proxy that enables function calls using the OpenAI interface, allowing it to be called from any library. This engineering workaround has proven to be immensely useful in my company for various projects. Here's the link to the GitHub repository where you can explore and play around with it. I've included some examples in this collaboration for you to check out. 15 | 16 | 17 | 18 | Powered by Groq 19 | 20 | 21 | 22 | ## Running the Proxy Locally 🖥️ 23 | To run this proxy locally on your own machine, follow these steps: 24 | 25 | 1. Clone the GitHub repository: 26 | ```git clone https://github.com/unclecode/groqcall.git``` 27 | 28 | 2. Navigate to the project directory: 29 | ```cd groqcall``` 30 | 31 | 3. Create a virtual environment: 32 | ```python -m venv venv``` 33 | 34 | 4. Activate virtual environment: 35 | ```source venv/bin/activate``` 36 | 37 | 5. Install the required libraries: 38 | ```pip install -r requirements.txt``` 39 | 40 | 6. Run the FastAPI server: 41 | ```./venv/bin/uvicorn --app-dir app/ main:app --reload``` 42 | 43 | 44 | ## Using the Pre-built Server 🌐 45 | For your convenience, I have already set up a server that you can use temporarily. This allows you to quickly start using the proxy without having to run it locally. 46 | 47 | To use the pre-built server, simply make requests to the following base URL: 48 | ```https://groqcall.ai/proxy/groq/v1``` 49 | 50 | 51 | ## Exploring GroqCall.ai 🚀 52 | This README is organized into three main sections, each showcasing different aspects of GroqCall.ai: 53 | 54 | - **Sending POST Requests**: Here, I explore the functionality of sending direct POST requests to LLMs using GroqCall.ai. This section highlights the flexibility and control offered by the library when interacting with LLMs. 55 | - **FuncHub**: The second section introduces the concept of FuncHub, a useful feature that simplifies the process of executing functions. With FuncHub, there is no need to send the function JSON schema explicitly, as the functions are already hosted on the proxy server. This approach streamlines the workflow, allowing developers to obtain results with a single call without having to handle function call is production server. 56 | - **Using GroqCall with PhiData**: In this section, I demonstrate how GroqCall.ai can be seamlessly integrated with other libraries such as my favorite one, the PhiData library, leveraging its built-in tools to connect to LLMs and perform external tool requests. 57 | 58 | 59 | ```python 60 | # The following libraries are optional if you're interested in using PhiData or managing your tools on the client side. 61 | !pip install phidata > /dev/null 62 | !pip install openai > /dev/null 63 | !pip install duckduckgo-search > /dev/null 64 | ``` 65 | 66 | ## Sending POST request, with full functions implementation 67 | 68 | 69 | ```python 70 | from duckduckgo_search import DDGS 71 | import requests, os 72 | import json 73 | 74 | # Here you pass your own GROQ API key 75 | api_key=userdata.get("GROQ_API_KEY") 76 | header = { 77 | "Authorization": f"Bearer {api_key}", 78 | "Content-Type": "application/json" 79 | } 80 | proxy_url = "https://groqcall.ai/proxy/groq/v1/chat/completions" 81 | 82 | 83 | def duckduckgo_search(query, max_results=None): 84 | """ 85 | Use this function to search DuckDuckGo for a query. 86 | """ 87 | with DDGS() as ddgs: 88 | return [r for r in ddgs.text(query, safesearch='off', max_results=max_results)] 89 | 90 | def duckduckgo_news(query, max_results=None): 91 | """ 92 | Use this function to get the latest news from DuckDuckGo. 93 | """ 94 | with DDGS() as ddgs: 95 | return [r for r in ddgs.news(query, safesearch='off', max_results=max_results)] 96 | 97 | function_map = { 98 | "duckduckgo_search": duckduckgo_search, 99 | "duckduckgo_news": duckduckgo_news, 100 | } 101 | 102 | request = { 103 | "messages": [ 104 | { 105 | "role": "system", 106 | "content": "YOU MUST FOLLOW THESE INSTRUCTIONS CAREFULLY.\n\n1. Use markdown to format your answers.\n" 107 | }, 108 | { 109 | "role": "user", 110 | "content": "Whats happening in France? Summarize top stories with sources, very short and concise." 111 | } 112 | ], 113 | "model": "mixtral-8x7b-32768", 114 | "tool_choice": "auto", 115 | "tools": [ 116 | { 117 | "type": "function", 118 | "function": { 119 | "name": "duckduckgo_search", 120 | "description": "Use this function to search DuckDuckGo for a query.\n\nArgs:\n query(str): The query to search for.\n max_results (optional, default=5): The maximum number of results to return.\n\nReturns:\n The result from DuckDuckGo.", 121 | "parameters": { 122 | "type": "object", 123 | "properties": { 124 | "query": { 125 | "type": "string" 126 | }, 127 | "max_results": { 128 | "type": [ 129 | "number", 130 | "null" 131 | ] 132 | } 133 | } 134 | } 135 | } 136 | }, 137 | { 138 | "type": "function", 139 | "function": { 140 | "name": "duckduckgo_news", 141 | "description": "Use this function to get the latest news from DuckDuckGo.\n\nArgs:\n query(str): The query to search for.\n max_results (optional, default=5): The maximum number of results to return.\n\nReturns:\n The latest news from DuckDuckGo.", 142 | "parameters": { 143 | "type": "object", 144 | "properties": { 145 | "query": { 146 | "type": "string" 147 | }, 148 | "max_results": { 149 | "type": [ 150 | "number", 151 | "null" 152 | ] 153 | } 154 | } 155 | } 156 | } 157 | } 158 | ] 159 | } 160 | 161 | response = requests.post( 162 | proxy_url, 163 | headers=header, 164 | json=request 165 | ) 166 | if response.status_code == 200: 167 | res = response.json() 168 | message = res['choices'][0]['message'] 169 | tools_response_messages = [] 170 | if not message['content'] and 'tool_calls' in message: 171 | for tool_call in message['tool_calls']: 172 | tool_name = tool_call['function']['name'] 173 | tool_args = tool_call['function']['arguments'] 174 | tool_args = json.loads(tool_args) 175 | if tool_name not in function_map: 176 | print(f"Error: {tool_name} is not a valid function name.") 177 | continue 178 | tool_func = function_map[tool_name] 179 | tool_response = tool_func(**tool_args) 180 | tools_response_messages.append({ 181 | "role": "tool", "content": json.dumps(tool_response) 182 | }) 183 | 184 | if tools_response_messages: 185 | request['messages'] += tools_response_messages 186 | response = requests.post( 187 | proxy_url, 188 | headers=header, 189 | json=request 190 | ) 191 | if response.status_code == 200: 192 | res = response.json() 193 | print(res['choices'][0]['message']['content']) 194 | else: 195 | print("Error:", response.status_code, response.text) 196 | else: 197 | print(message['content']) 198 | else: 199 | print("Error:", response.status_code, response.text) 200 | 201 | ``` 202 | 203 | ## Schema-less Function Call 🤩 204 | In this method, we only need to provide the function's name, which consists of two parts, acting as a sort of namespace. The first part identifies the library or toolkit containing the functions, and the second part specifies the function's name, assuming it's already available on the proxy server. I aim to collaborate with the community to incorporate all typical functions, eliminating the need for passing a schema. Without having to handle function calls ourselves, a single request to the proxy enables it to identify and execute the functions, retrieve responses from large language models, and return the results to us. Thanks to Groq, all of this occurs in just seconds. 205 | 206 | 207 | ```python 208 | from duckduckgo_search import DDGS 209 | import requests, os 210 | api_key = userdata.get("GROQ_API_KEY") 211 | header = { 212 | "Authorization": f"Bearer {api_key}", 213 | "Content-Type": "application/json" 214 | } 215 | 216 | proxy_url = "https://groqcall.ai/proxy/groq/v1/chat/completions" 217 | 218 | 219 | request = { 220 | "messages": [ 221 | { 222 | "role": "system", 223 | "content": "YOU MUST FOLLOW THESE INSTRUCTIONS CAREFULLY.\n\n1. Use markdown to format your answers.\n", 224 | }, 225 | { 226 | "role": "user", 227 | "content": "Whats happening in France? Summarize top stories with sources, very short and concise. Also please search about the histoy of france as well.", 228 | }, 229 | ], 230 | "model": "mixtral-8x7b-32768", 231 | "tool_choice": "auto", 232 | "tools": [ 233 | { 234 | "type": "function", 235 | "function": { 236 | "name": "duckduck.search", 237 | }, 238 | }, 239 | { 240 | "type": "function", 241 | "function": { 242 | "name": "duckduck.news", 243 | }, 244 | }, 245 | ], 246 | } 247 | 248 | response = requests.post( 249 | proxy_url, 250 | headers=header, 251 | json=request, 252 | ) 253 | 254 | if response.status_code == 200: 255 | res = response.json() 256 | print(res["choices"][0]["message"]["content"]) 257 | else: 258 | print("Error:", response.status_code, response.text) 259 | 260 | ``` 261 | 262 | ## Using with PhiData 263 | FindData is a favorite of mine for creating AI assistants, thanks to its beautifully simplified interface, unlike the complexity seen in the LangChain library and LlamaIndex. I use it for many projects and want to give kudos to their team. It's open source, and I recommend everyone check it out. You can explore more from this link https://github.com/phidatahq/phidata. 264 | 265 | 266 | ```python 267 | from google.README import userdata 268 | from phi.llm.openai.like import OpenAILike 269 | from phi.assistant import Assistant 270 | from phi.tools.duckduckgo import DuckDuckGo 271 | import os, json 272 | 273 | 274 | my_groq = OpenAILike( 275 | model="mixtral-8x7b-32768", 276 | api_key=userdata.get("GROQ_API_KEY"), 277 | base_url="https://groqcall.ai/proxy/groq/v1" 278 | ) 279 | assistant = Assistant( 280 | llm=my_groq, 281 | tools=[DuckDuckGo()], show_tool_calls=True, markdown=True 282 | ) 283 | assistant.print_response("Whats happening in France? Summarize top stories with sources, very short and concise.", stream=False) 284 | 285 | 286 | ``` 287 | 288 | ## Contributions Welcome! 🙌 289 | I am excited to extend and grow this repository by adding more built-in functions and integrating additional services. If you are interested in contributing to this project and being a part of its development, I would love to collaborate with you! I plan to create a discord channel for this project, where we can discuss ideas, share knowledge, and work together to enhance the repository. 290 | 291 | Here's how you can get involved: 292 | 293 | 1. Fork the repository and create your own branch. 294 | 2. Implement new functions, integrate additional services, or make improvements to the existing codebase. 295 | 3. Test your changes to ensure they work as expected. 296 | 4. Submit a pull request describing the changes you have made and why they are valuable. 297 | 298 | If you have any ideas, suggestions, or would like to discuss potential contributions, feel free to reach out to me. You can contact me through the following channels: 299 | 300 | - Twitter (X): @unclecode 301 | - Email: unclecode@kidocode.com 302 | 303 | ### Copyright 2024 Unclecode (Hossein Tohidi) 304 | 305 | Licensed under the Apache License, Version 2.0 (the "License"); 306 | you may not use this file except in compliance with the License. 307 | You may obtain a copy of the License at 308 | 309 | http://www.apache.org/licenses/LICENSE-2.0 310 | 311 | Unless required by applicable law or agreed to in writing, software 312 | distributed under the License is distributed on an "AS IS" BASIS, 313 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 314 | See the License for the specific language governing permissions and 315 | limitations under the License. 316 | 317 | I'm open to collaboration and excited to see how we can work together to enhance this project and provide value to the community. Let's connect and explore how we can help each other! 318 | 319 | Together, let's make this repository even more awesome! 🚀 320 | -------------------------------------------------------------------------------- /app/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unclecode/groqcall/009c301a8a7125fba5a5e7eb0e0c0b1fba4aa211/app/__init__.py -------------------------------------------------------------------------------- /app/config.py: -------------------------------------------------------------------------------- 1 | # To be developed 2 | EVALUATION_CYCLES_COUNT=1 3 | PARSE_ERROR_TRIES = 5 -------------------------------------------------------------------------------- /app/functions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unclecode/groqcall/009c301a8a7125fba5a5e7eb0e0c0b1fba4aa211/app/functions/__init__.py -------------------------------------------------------------------------------- /app/functions/base.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | from typing import Dict 3 | 4 | class Function: 5 | name: str 6 | description: str 7 | 8 | class Schema(BaseModel): 9 | pass 10 | 11 | @classmethod 12 | def get_schema(cls) -> Dict: 13 | schema_dict = { 14 | "name": cls.name, 15 | "description": cls.description, 16 | "parameters": cls.Schema.schema(), 17 | } 18 | return schema_dict -------------------------------------------------------------------------------- /app/functions/duckduck.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel, Field 2 | from typing import Optional, Dict 3 | import requests 4 | import json 5 | from duckduckgo_search import DDGS 6 | 7 | # from .base import Function 8 | from pydantic import Field 9 | from typing import Optional 10 | import requests 11 | import json 12 | 13 | from .base import Function 14 | 15 | class SearchFunction(Function): 16 | name = "duckduck.search" 17 | description = "Use this function to search DuckDuckGo for a query.\n\nArgs:\n query(str): The query to search for.\n max_results (optional, default=5): The maximum number of results to return.\n\nReturns:\n The result from DuckDuckGo." 18 | 19 | class Schema(Function.Schema): 20 | query: str = Field(..., description="The query to search for.") 21 | max_results: Optional[int] = Field(5, description="The maximum number of results to return.") 22 | 23 | @classmethod 24 | def run(cls, **kwargs): 25 | query = kwargs.get("query") 26 | max_results = kwargs.get("max_results") 27 | with DDGS() as ddgs: 28 | return [r for r in ddgs.text(query, safesearch='off', max_results=max_results)] 29 | 30 | 31 | class NewsFunction(Function): 32 | name = "duckduck.news" 33 | description = "Use this function to get the latest news from DuckDuckGo.\n\nArgs:\n query(str): The query to search for.\n max_results (optional, default=5): The maximum number of results to return.\n\nReturns:\n The latest news from DuckDuckGo." 34 | 35 | class Schema(Function.Schema): 36 | query: str = Field(..., description="The query to search for.") 37 | max_results: Optional[int] = Field(5, description="The maximum number of results to return.") 38 | 39 | @classmethod 40 | def run(cls, **kwargs): 41 | query = kwargs.get("query") 42 | max_results = kwargs.get("max_results") 43 | 44 | with DDGS() as ddgs: 45 | results = [r for r in ddgs.news(query, safesearch='off', max_results=max_results)] 46 | return results -------------------------------------------------------------------------------- /app/libs/__init__.py: -------------------------------------------------------------------------------- 1 | from .context import Context 2 | from .base_handler import Handler, DefaultCompletionHandler, ExceptionHandler, FallbackHandler 3 | from .provider_handler import ProviderSelectionHandler 4 | from .vision_handler import ImageMessageHandler 5 | from .tools_handler import ToolExtractionHandler, ToolResponseHandler 6 | 7 | __all__ = [ 8 | "Context", 9 | "Handler", 10 | "DefaultCompletionHandler", 11 | "ExceptionHandler", 12 | "ProviderSelectionHandler", 13 | "ImageMessageHandler", 14 | "ToolExtractionHandler", 15 | "ToolResponseHandler", 16 | "FallbackHandler", 17 | ] 18 | -------------------------------------------------------------------------------- /app/libs/base_handler.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from .context import Context 3 | from fastapi.responses import JSONResponse 4 | import traceback 5 | 6 | class Handler(ABC): 7 | """Abstract Handler class for building the chain of handlers.""" 8 | 9 | _next_handler: "Handler" = None 10 | 11 | def set_next(self, handler: "Handler") -> "Handler": 12 | self._next_handler = handler 13 | return handler 14 | 15 | @abstractmethod 16 | async def handle(self, context: Context): 17 | if self._next_handler: 18 | try: 19 | return await self._next_handler.handle(context) 20 | except Exception as e: 21 | _exception_handler: "Handler" = ExceptionHandler() 22 | # Extract the stack trace and log the exception 23 | return await _exception_handler.handle(self._next_handler, context, e) 24 | 25 | 26 | class DefaultCompletionHandler(Handler): 27 | async def handle(self, context: Context): 28 | if context.is_normal_chat: 29 | # Assuming context.client is set and has a method for creating chat completions 30 | completion = context.client.route( 31 | messages=context.messages, 32 | **context.client.clean_params(context.params), 33 | ) 34 | context.response = completion.model_dump() 35 | return JSONResponse(content=context.response, status_code=200) 36 | 37 | return await super().handle(context) 38 | 39 | 40 | class FallbackHandler(Handler): 41 | async def handle(self, context: Context): 42 | # This handler does not pass the request further down the chain. 43 | # It acts as a fallback when no other handler has processed the request. 44 | if not context.response: 45 | # The default action when no other handlers have processed the request 46 | context.response = {"message": "No suitable action found for the request."} 47 | return JSONResponse(content=context.response, status_code=400) 48 | 49 | # If there's already a response set in the context, it means one of the handlers has processed the request. 50 | return JSONResponse(content=context.response, status_code=200) 51 | 52 | 53 | class ExceptionHandler(Handler): 54 | async def handle(self, handler: Handler, context: Context, exception: Exception): 55 | print(f"Error processing the request: {str(handler.__class__) } - {exception}") 56 | # print(traceback.format_exc()) 57 | return JSONResponse( 58 | content={"error": "An unexpected error occurred, within handler " + str(handler.__class__) + " : " + str(exception)}, 59 | status_code=500, 60 | ) 61 | 62 | 63 | -------------------------------------------------------------------------------- /app/libs/chains copy.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from typing import Any, Dict 3 | from importlib import import_module 4 | import json 5 | import uuid 6 | import traceback 7 | from fastapi import Request 8 | from fastapi.responses import JSONResponse 9 | from providers import BaseProvider 10 | from prompts import * 11 | from providers import GroqProvider 12 | import importlib 13 | from utils import get_tool_call_response, create_logger, describe 14 | 15 | missed_tool_logger = create_logger( 16 | "chain.missed_tools", ".logs/empty_tool_tool_response.log" 17 | ) 18 | 19 | 20 | class Context: 21 | def __init__(self, request: Request, provider: str, body: Dict[str, Any]): 22 | self.request = request 23 | self.provider = provider 24 | self.body = body 25 | self.response = None 26 | 27 | # extract all keys from body except messages and tools and set in params 28 | self.params = {k: v for k, v in body.items() if k not in ["messages", "tools"]} 29 | 30 | # self.no_tool_behaviour = self.params.get("no_tool_behaviour", "return") 31 | self.no_tool_behaviour = self.params.get("no_tool_behaviour", "forward") 32 | self.params.pop("no_tool_behaviour", None) 33 | 34 | # Todo: For now, no stream, sorry ;) 35 | self.params["stream"] = False 36 | 37 | self.messages = body.get("messages", []) 38 | self.tools = body.get("tools", []) 39 | 40 | self.builtin_tools = [ 41 | t for t in self.tools if "parameters" not in t["function"] 42 | ] 43 | self.builtin_tool_names = [t["function"]["name"] for t in self.builtin_tools] 44 | self.custom_tools = [t for t in self.tools if "parameters" in t["function"]] 45 | 46 | for bt in self.builtin_tools: 47 | func_namespace = bt["function"]["name"] 48 | if len(func_namespace.split(".")) == 2: 49 | module_name, func_class_name = func_namespace.split(".") 50 | func_class_name = f"{func_class_name.capitalize()}Function" 51 | # raise ValueError("Only one builtin function can be called at a time.") 52 | module = importlib.import_module(f"app.functions.{module_name}") 53 | func_class = getattr(module, func_class_name, None) 54 | schema_dict = func_class.get_schema() 55 | if schema_dict: 56 | bt["function"] = schema_dict 57 | bt["run"] = func_class.run 58 | bt["extra"] = self.params.get("extra", {}) 59 | self.params.pop("extra", None) 60 | 61 | self.client: BaseProvider = None 62 | 63 | @property 64 | def last_message(self): 65 | return self.messages[-1] if self.messages else {} 66 | 67 | @property 68 | def is_tool_call(self): 69 | return bool( 70 | self.last_message["role"] == "user" 71 | and self.tools 72 | and self.params.get("tool_choice", "none") != "none" 73 | ) 74 | 75 | @property 76 | def is_tool_response(self): 77 | return bool(self.last_message["role"] == "tool" and self.tools) 78 | 79 | @property 80 | def is_normal_chat(self): 81 | return bool(not self.is_tool_call and not self.is_tool_response) 82 | 83 | 84 | class Handler(ABC): 85 | """Abstract Handler class for building the chain of handlers.""" 86 | 87 | _next_handler: "Handler" = None 88 | 89 | def set_next(self, handler: "Handler") -> "Handler": 90 | self._next_handler = handler 91 | return handler 92 | 93 | @abstractmethod 94 | async def handle(self, context: Context): 95 | if self._next_handler: 96 | try: 97 | return await self._next_handler.handle(context) 98 | except Exception as e: 99 | _exception_handler: "Handler" = ExceptionHandler() 100 | # Extract the stack trace and log the exception 101 | return await _exception_handler.handle(context, e) 102 | 103 | 104 | class ProviderSelectionHandler(Handler): 105 | @staticmethod 106 | def provider_exists(provider: str) -> bool: 107 | module_name = f"app.providers" 108 | class_name = f"{provider.capitalize()}Provider" 109 | try: 110 | provider_module = import_module(module_name) 111 | provider_class = getattr(provider_module, class_name) 112 | return bool(provider_class) 113 | except ImportError: 114 | return False 115 | 116 | async def handle(self, context: Context): 117 | # Construct the module path and class name based on the provider 118 | module_name = f"app.providers" 119 | class_name = f"{context.provider.capitalize()}Provider" 120 | 121 | try: 122 | # Dynamically import the module and class 123 | provider_module = import_module(module_name) 124 | provider_class = getattr(provider_module, class_name) 125 | 126 | if provider_class: 127 | context.client = provider_class( 128 | api_key=context.api_token 129 | ) # Assuming an api_key parameter 130 | return await super().handle(context) 131 | else: 132 | raise ValueError( 133 | f"Provider class {class_name} could not be found in {module_name}." 134 | ) 135 | except ImportError as e: 136 | # Handle import error (e.g., module or class not found) 137 | print(f"Error importing {class_name} from {module_name}: {e}") 138 | context.response = { 139 | "error": f"An error occurred while trying to load the provider: {e}" 140 | } 141 | return JSONResponse(content=context.response, status_code=500) 142 | 143 | 144 | class ImageMessageHandler(Handler): 145 | async def handle(self, context: Context): 146 | new_messages = [] 147 | image_ref = 1 148 | for message in context.messages: 149 | if message["role"] == "user": 150 | if isinstance(message["content"], list): 151 | prompt = None 152 | for content in message["content"]: 153 | if content["type"] == "text": 154 | # new_messages.append({"role": message["role"], "content": content["text"]}) 155 | prompt = content["text"] 156 | elif content["type"] == "image_url": 157 | image_url = content["image_url"]["url"] 158 | try: 159 | prompt = prompt or IMAGE_DESCRIPTO_PROMPT 160 | description = describe(prompt, image_url) 161 | if description: 162 | description = get_image_desc_guide(image_ref, description) 163 | new_messages.append( 164 | {"role": message["role"], "content": description} 165 | ) 166 | image_ref += 1 167 | else: 168 | pass 169 | except Exception as e: 170 | print(f"Error describing image: {e}") 171 | continue 172 | else: 173 | new_messages.append(message) 174 | else: 175 | new_messages.append(message) 176 | 177 | context.messages = new_messages 178 | return await super().handle(context) 179 | 180 | 181 | class ImageLLavaMessageHandler(Handler): 182 | async def handle(self, context: Context): 183 | new_messages = [] 184 | image_ref = 1 185 | for message in context.messages: 186 | new_messages.append(message) 187 | if message["role"] == "user": 188 | if isinstance(message["content"], list): 189 | for content in message["content"]: 190 | if content["type"] == "text": 191 | prompt = content["text"] 192 | elif content["type"] == "image_url": 193 | image_url = content["image_url"]["url"] 194 | try: 195 | description = describe(prompt, image_url) 196 | new_messages.append( 197 | {"role": "assistant", "content": description} 198 | ) 199 | image_ref += 1 200 | except Exception as e: 201 | print(f"Error describing image: {e}") 202 | continue 203 | context.messages = new_messages 204 | return await super().handle(context) 205 | 206 | 207 | class ToolExtractionHandler(Handler): 208 | async def handle(self, context: Context): 209 | body = context.body 210 | if context.is_tool_call: 211 | 212 | # Prepare the messages and tools for the tool extraction 213 | messages = [ 214 | f"{m['role'].title()}: {m['content']}" 215 | for m in context.messages 216 | if m["role"] != "system" 217 | ] 218 | tools_json = json.dumps([t["function"] for t in context.tools], indent=4) 219 | 220 | # Process the tool_choice 221 | tool_choice = context.params.get("tool_choice", "auto") 222 | forced_mode = False 223 | if ( 224 | type(tool_choice) == dict 225 | and tool_choice.get("type", None) == "function" 226 | ): 227 | tool_choice = tool_choice["function"].get("name", None) 228 | if not tool_choice: 229 | raise ValueError( 230 | "Invalid tool choice. 'tool_choice' is set to a dictionary with 'type' as 'function', but 'function' does not have a 'name' key." 231 | ) 232 | forced_mode = True 233 | 234 | # Regenerate the string tool_json and keep only the forced tool 235 | tools_json = json.dumps( 236 | [ 237 | t["function"] 238 | for t in context.tools 239 | if t["function"]["name"] == tool_choice 240 | ], 241 | indent=4, 242 | ) 243 | 244 | system_message = ( 245 | SYSTEM_MESSAGE if not forced_mode else ENFORCED_SYSTAME_MESSAE 246 | ) 247 | suffix = SUFFIX if not forced_mode else get_forced_tool_suffix(tool_choice) 248 | 249 | new_messages = [ 250 | {"role": "system", "content": system_message}, 251 | { 252 | "role": "system", 253 | "content": f"Conversation History:\n{''.join(messages)}\n\nTools: \n{tools_json}\n\n{suffix}", 254 | }, 255 | ] 256 | 257 | completion, tool_calls = await self.process_tool_calls( 258 | context, new_messages 259 | ) 260 | 261 | if not tool_calls: 262 | if context.no_tool_behaviour == "forward": 263 | context.tools = None 264 | return await super().handle(context) 265 | else: 266 | context.response = {"tool_calls": []} 267 | tool_response = get_tool_call_response(completion, [], []) 268 | missed_tool_logger.debug( 269 | f"Last message content: {context.last_message['content']}" 270 | ) 271 | return JSONResponse(content=tool_response, status_code=200) 272 | 273 | unresolved_tol_calls = [ 274 | t 275 | for t in tool_calls 276 | if t["function"]["name"] not in context.builtin_tool_names 277 | ] 278 | resolved_responses = [] 279 | for tool in tool_calls: 280 | for bt in context.builtin_tools: 281 | if tool["function"]["name"] == bt["function"]["name"]: 282 | res = bt["run"]( 283 | **{ 284 | **json.loads(tool["function"]["arguments"]), 285 | **bt["extra"], 286 | } 287 | ) 288 | resolved_responses.append( 289 | { 290 | "name": tool["function"]["name"], 291 | "role": "tool", 292 | "content": json.dumps(res), 293 | "tool_call_id": "chatcmpl-" + completion.id, 294 | } 295 | ) 296 | 297 | if not unresolved_tol_calls: 298 | context.messages.extend(resolved_responses) 299 | return await super().handle(context) 300 | 301 | tool_response = get_tool_call_response( 302 | completion, unresolved_tol_calls, resolved_responses 303 | ) 304 | 305 | context.response = tool_response 306 | return JSONResponse(content=context.response, status_code=200) 307 | 308 | return await super().handle(context) 309 | 310 | async def process_tool_calls(self, context, new_messages): 311 | try: 312 | tries = 5 313 | tool_calls = [] 314 | while tries > 0: 315 | try: 316 | # Assuming the context has an instantiated client according to the selected provider 317 | completion = context.client.route( 318 | model=context.client.parser_model, 319 | messages=new_messages, 320 | temperature=0, 321 | max_tokens=1024, 322 | top_p=1, 323 | stream=False, 324 | ) 325 | 326 | response = completion.choices[0].message.content 327 | if "```json" in response: 328 | response = response.split("```json")[1].split("```")[0] 329 | 330 | try: 331 | tool_response = json.loads(response) 332 | if isinstance(tool_response, list): 333 | tool_response = {"tool_calls": tool_response} 334 | except json.JSONDecodeError as e: 335 | print( 336 | f"Error parsing the tool response: {e}, tries left: {tries}" 337 | ) 338 | new_messages.append( 339 | { 340 | "role": "user", 341 | "content": f"Error: {e}.\n\n{CLEAN_UP_MESSAGE}", 342 | } 343 | ) 344 | tries -= 1 345 | continue 346 | 347 | for func in tool_response.get("tool_calls", []): 348 | tool_calls.append( 349 | { 350 | "id": f"call_{func['name']}_{str(uuid.uuid4())}", 351 | "type": "function", 352 | "function": { 353 | "name": func["name"], 354 | "arguments": json.dumps(func["arguments"]), 355 | }, 356 | } 357 | ) 358 | 359 | break 360 | except Exception as e: 361 | raise e 362 | 363 | if tries == 0: 364 | tool_calls = [] 365 | 366 | return completion, tool_calls 367 | except Exception as e: 368 | print(f"Error processing the tool calls: {e}") 369 | raise e 370 | 371 | 372 | class ToolResponseHandler(Handler): 373 | async def handle(self, context: Context): 374 | body = context.body 375 | if context.is_tool_response: 376 | messages = context.messages 377 | 378 | for message in messages: 379 | if message["role"] == "tool": 380 | message["role"] = "user" 381 | message["content"] = get_func_result_guide(message["content"]) 382 | 383 | messages[-1]["role"] = "user" 384 | # Assuming get_func_result_guide is a function that formats the tool response 385 | messages[-1]["content"] = get_func_result_guide(messages[-1]["content"]) 386 | 387 | try: 388 | completion = context.client.route( 389 | messages=messages, 390 | **context.client.clean_params(context.params), 391 | ) 392 | context.response = completion.model_dump() 393 | return JSONResponse(content=context.response, status_code=200) 394 | except Exception as e: 395 | # Log the exception or handle it as needed 396 | print(e) 397 | context.response = { 398 | "error": "An error occurred processing the tool response" 399 | } 400 | return JSONResponse(content=context.response, status_code=500) 401 | 402 | return await super().handle(context) 403 | 404 | 405 | class DefaultCompletionHandler(Handler): 406 | async def handle(self, context: Context): 407 | if context.is_normal_chat: 408 | # Assuming context.client is set and has a method for creating chat completions 409 | completion = context.client.route( 410 | messages=context.messages, 411 | **context.client.clean_params(context.params), 412 | ) 413 | context.response = completion.model_dump() 414 | return JSONResponse(content=context.response, status_code=200) 415 | 416 | return await super().handle(context) 417 | 418 | 419 | class FallbackHandler(Handler): 420 | async def handle(self, context: Context): 421 | # This handler does not pass the request further down the chain. 422 | # It acts as a fallback when no other handler has processed the request. 423 | if not context.response: 424 | # The default action when no other handlers have processed the request 425 | context.response = {"message": "No suitable action found for the request."} 426 | return JSONResponse(content=context.response, status_code=400) 427 | 428 | # If there's already a response set in the context, it means one of the handlers has processed the request. 429 | return JSONResponse(content=context.response, status_code=200) 430 | 431 | 432 | class ExceptionHandler(Handler): 433 | async def handle(self, context: Context, exception: Exception): 434 | print(f"Error processing the request: {exception}") 435 | print(traceback.format_exc()) 436 | return JSONResponse( 437 | content={"error": "An unexpected error occurred. " + str(exception)}, 438 | status_code=500, 439 | ) 440 | -------------------------------------------------------------------------------- /app/libs/chains.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from typing import Any, Dict 3 | from importlib import import_module 4 | import json 5 | import uuid 6 | import traceback 7 | from fastapi import Request 8 | from fastapi.responses import JSONResponse 9 | from providers import BaseProvider 10 | from prompts import * 11 | from providers import GroqProvider 12 | import importlib 13 | from utils import get_tool_call_response, create_logger, describe 14 | 15 | missed_tool_logger = create_logger( 16 | "chain.missed_tools", ".logs/empty_tool_tool_response.log" 17 | ) 18 | 19 | 20 | class Context: 21 | def __init__(self, request: Request, provider: str, body: Dict[str, Any]): 22 | self.request = request 23 | self.provider = provider 24 | self.body = body 25 | self.response = None 26 | 27 | # extract all keys from body except messages and tools and set in params 28 | self.params = {k: v for k, v in body.items() if k not in ["messages", "tools"]} 29 | 30 | # self.no_tool_behaviour = self.params.get("no_tool_behaviour", "return") 31 | self.no_tool_behaviour = self.params.get("no_tool_behaviour", "forward") 32 | self.params.pop("no_tool_behaviour", None) 33 | 34 | # Todo: For now, no stream, sorry ;) 35 | self.params["stream"] = False 36 | 37 | self.messages = body.get("messages", []) 38 | self.tools = body.get("tools", []) 39 | 40 | self.builtin_tools = [ 41 | t for t in self.tools if "parameters" not in t["function"] 42 | ] 43 | self.builtin_tool_names = [t["function"]["name"] for t in self.builtin_tools] 44 | self.custom_tools = [t for t in self.tools if "parameters" in t["function"]] 45 | 46 | for bt in self.builtin_tools: 47 | func_namespace = bt["function"]["name"] 48 | if len(func_namespace.split(".")) == 2: 49 | module_name, func_class_name = func_namespace.split(".") 50 | func_class_name = f"{func_class_name.capitalize()}Function" 51 | # raise ValueError("Only one builtin function can be called at a time.") 52 | module = importlib.import_module(f"app.functions.{module_name}") 53 | func_class = getattr(module, func_class_name, None) 54 | schema_dict = func_class.get_schema() 55 | if schema_dict: 56 | bt["function"] = schema_dict 57 | bt["run"] = func_class.run 58 | bt["extra"] = self.params.get("extra", {}) 59 | self.params.pop("extra", None) 60 | 61 | self.client: BaseProvider = None 62 | 63 | @property 64 | def last_message(self): 65 | return self.messages[-1] if self.messages else {} 66 | 67 | @property 68 | def is_tool_call(self): 69 | return bool( 70 | self.last_message["role"] == "user" 71 | and self.tools 72 | and self.params.get("tool_choice", "none") != "none" 73 | ) 74 | 75 | @property 76 | def is_tool_response(self): 77 | return bool(self.last_message["role"] == "tool" and self.tools) 78 | 79 | @property 80 | def is_normal_chat(self): 81 | return bool(not self.is_tool_call and not self.is_tool_response) 82 | 83 | 84 | class Handler(ABC): 85 | """Abstract Handler class for building the chain of handlers.""" 86 | 87 | _next_handler: "Handler" = None 88 | 89 | def set_next(self, handler: "Handler") -> "Handler": 90 | self._next_handler = handler 91 | return handler 92 | 93 | @abstractmethod 94 | async def handle(self, context: Context): 95 | if self._next_handler: 96 | try: 97 | return await self._next_handler.handle(context) 98 | except Exception as e: 99 | _exception_handler: "Handler" = ExceptionHandler() 100 | # Extract the stack trace and log the exception 101 | return await _exception_handler.handle(context, e) 102 | 103 | 104 | class ProviderSelectionHandler(Handler): 105 | @staticmethod 106 | def provider_exists(provider: str) -> bool: 107 | module_name = f"app.providers" 108 | class_name = f"{provider.capitalize()}Provider" 109 | try: 110 | provider_module = import_module(module_name) 111 | provider_class = getattr(provider_module, class_name) 112 | return bool(provider_class) 113 | except ImportError: 114 | return False 115 | 116 | async def handle(self, context: Context): 117 | # Construct the module path and class name based on the provider 118 | module_name = f"app.providers" 119 | class_name = f"{context.provider.capitalize()}Provider" 120 | 121 | try: 122 | # Dynamically import the module and class 123 | provider_module = import_module(module_name) 124 | provider_class = getattr(provider_module, class_name) 125 | 126 | if provider_class: 127 | context.client = provider_class( 128 | api_key=context.api_token 129 | ) # Assuming an api_key parameter 130 | return await super().handle(context) 131 | else: 132 | raise ValueError( 133 | f"Provider class {class_name} could not be found in {module_name}." 134 | ) 135 | except ImportError as e: 136 | # Handle import error (e.g., module or class not found) 137 | print(f"Error importing {class_name} from {module_name}: {e}") 138 | context.response = { 139 | "error": f"An error occurred while trying to load the provider: {e}" 140 | } 141 | return JSONResponse(content=context.response, status_code=500) 142 | 143 | 144 | class ImageMessageHandler(Handler): 145 | async def handle(self, context: Context): 146 | new_messages = [] 147 | image_ref = 1 148 | for message in context.messages: 149 | if message["role"] == "user": 150 | if isinstance(message["content"], list): 151 | prompt = None 152 | for content in message["content"]: 153 | if content["type"] == "text": 154 | # new_messages.append({"role": message["role"], "content": content["text"]}) 155 | prompt = content["text"] 156 | elif content["type"] == "image_url": 157 | image_url = content["image_url"]["url"] 158 | try: 159 | prompt = prompt or IMAGE_DESCRIPTO_PROMPT 160 | description = describe(prompt, image_url) 161 | if description: 162 | description = get_image_desc_guide(image_ref, description) 163 | new_messages.append( 164 | {"role": message["role"], "content": description} 165 | ) 166 | image_ref += 1 167 | else: 168 | pass 169 | except Exception as e: 170 | print(f"Error describing image: {e}") 171 | continue 172 | else: 173 | new_messages.append(message) 174 | else: 175 | new_messages.append(message) 176 | 177 | context.messages = new_messages 178 | return await super().handle(context) 179 | 180 | 181 | class ImageLLavaMessageHandler(Handler): 182 | async def handle(self, context: Context): 183 | new_messages = [] 184 | image_ref = 1 185 | for message in context.messages: 186 | new_messages.append(message) 187 | if message["role"] == "user": 188 | if isinstance(message["content"], list): 189 | for content in message["content"]: 190 | if content["type"] == "text": 191 | prompt = content["text"] 192 | elif content["type"] == "image_url": 193 | image_url = content["image_url"]["url"] 194 | try: 195 | description = describe(prompt, image_url) 196 | new_messages.append( 197 | {"role": "assistant", "content": description} 198 | ) 199 | image_ref += 1 200 | except Exception as e: 201 | print(f"Error describing image: {e}") 202 | continue 203 | context.messages = new_messages 204 | return await super().handle(context) 205 | 206 | 207 | class ToolExtractionHandler(Handler): 208 | async def handle(self, context: Context): 209 | body = context.body 210 | if context.is_tool_call: 211 | 212 | # Prepare the messages and tools for the tool extraction 213 | messages = [ 214 | f"{m['role'].title()}: {m['content']}" 215 | for m in context.messages 216 | if m["role"] != "system" 217 | ] 218 | tools_json = json.dumps([t["function"] for t in context.tools], indent=4) 219 | 220 | # Process the tool_choice 221 | tool_choice = context.params.get("tool_choice", "auto") 222 | forced_mode = False 223 | if ( 224 | type(tool_choice) == dict 225 | and tool_choice.get("type", None) == "function" 226 | ): 227 | tool_choice = tool_choice["function"].get("name", None) 228 | if not tool_choice: 229 | raise ValueError( 230 | "Invalid tool choice. 'tool_choice' is set to a dictionary with 'type' as 'function', but 'function' does not have a 'name' key." 231 | ) 232 | forced_mode = True 233 | 234 | # Regenerate the string tool_json and keep only the forced tool 235 | tools_json = json.dumps( 236 | [ 237 | t["function"] 238 | for t in context.tools 239 | if t["function"]["name"] == tool_choice 240 | ], 241 | indent=4, 242 | ) 243 | 244 | system_message = ( 245 | SYSTEM_MESSAGE if not forced_mode else ENFORCED_SYSTAME_MESSAE 246 | ) 247 | suffix = SUFFIX if not forced_mode else get_forced_tool_suffix(tool_choice) 248 | 249 | new_messages = [ 250 | {"role": "system", "content": system_message}, 251 | { 252 | "role": "system", 253 | "content": f"Conversation History:\n{''.join(messages)}\n\nTools: \n{tools_json}\n\n{suffix}", 254 | }, 255 | ] 256 | 257 | completion, tool_calls = await self.process_tool_calls( 258 | context, new_messages 259 | ) 260 | 261 | if not tool_calls: 262 | if context.no_tool_behaviour == "forward": 263 | context.tools = None 264 | return await super().handle(context) 265 | else: 266 | context.response = {"tool_calls": []} 267 | tool_response = get_tool_call_response(completion, [], []) 268 | missed_tool_logger.debug( 269 | f"Last message content: {context.last_message['content']}" 270 | ) 271 | return JSONResponse(content=tool_response, status_code=200) 272 | 273 | unresolved_tol_calls = [ 274 | t 275 | for t in tool_calls 276 | if t["function"]["name"] not in context.builtin_tool_names 277 | ] 278 | resolved_responses = [] 279 | for tool in tool_calls: 280 | for bt in context.builtin_tools: 281 | if tool["function"]["name"] == bt["function"]["name"]: 282 | res = bt["run"]( 283 | **{ 284 | **json.loads(tool["function"]["arguments"]), 285 | **bt["extra"], 286 | } 287 | ) 288 | resolved_responses.append( 289 | { 290 | "name": tool["function"]["name"], 291 | "role": "tool", 292 | "content": json.dumps(res), 293 | "tool_call_id": "chatcmpl-" + completion.id, 294 | } 295 | ) 296 | 297 | if not unresolved_tol_calls: 298 | context.messages.extend(resolved_responses) 299 | return await super().handle(context) 300 | 301 | tool_response = get_tool_call_response( 302 | completion, unresolved_tol_calls, resolved_responses 303 | ) 304 | 305 | context.response = tool_response 306 | return JSONResponse(content=context.response, status_code=200) 307 | 308 | return await super().handle(context) 309 | 310 | async def process_tool_calls(self, context, new_messages): 311 | try: 312 | tries = 5 313 | tool_calls = [] 314 | while tries > 0: 315 | try: 316 | # Assuming the context has an instantiated client according to the selected provider 317 | completion = context.client.route( 318 | model=context.client.parser_model, 319 | messages=new_messages, 320 | temperature=0, 321 | max_tokens=1024, 322 | top_p=1, 323 | stream=False, 324 | ) 325 | 326 | response = completion.choices[0].message.content 327 | if "```json" in response: 328 | response = response.split("```json")[1].split("```")[0] 329 | 330 | try: 331 | tool_response = json.loads(response) 332 | if isinstance(tool_response, list): 333 | tool_response = {"tool_calls": tool_response} 334 | except json.JSONDecodeError as e: 335 | print( 336 | f"Error parsing the tool response: {e}, tries left: {tries}" 337 | ) 338 | new_messages.append( 339 | { 340 | "role": "user", 341 | "content": f"Error: {e}.\n\n{CLEAN_UP_MESSAGE}", 342 | } 343 | ) 344 | tries -= 1 345 | continue 346 | 347 | for func in tool_response.get("tool_calls", []): 348 | tool_calls.append( 349 | { 350 | "id": f"call_{func['name']}_{str(uuid.uuid4())}", 351 | "type": "function", 352 | "function": { 353 | "name": func["name"], 354 | "arguments": json.dumps(func["arguments"]), 355 | }, 356 | } 357 | ) 358 | 359 | break 360 | except Exception as e: 361 | raise e 362 | 363 | if tries == 0: 364 | tool_calls = [] 365 | 366 | return completion, tool_calls 367 | except Exception as e: 368 | print(f"Error processing the tool calls: {e}") 369 | raise e 370 | 371 | 372 | class ToolResponseHandler(Handler): 373 | async def handle(self, context: Context): 374 | body = context.body 375 | if context.is_tool_response: 376 | messages = context.messages 377 | 378 | for message in messages: 379 | if message["role"] == "tool": 380 | message["role"] = "user" 381 | message["content"] = get_func_result_guide(message["content"]) 382 | 383 | messages[-1]["role"] = "user" 384 | # Assuming get_func_result_guide is a function that formats the tool response 385 | messages[-1]["content"] = get_func_result_guide(messages[-1]["content"]) 386 | 387 | try: 388 | completion = context.client.route( 389 | messages=messages, 390 | **context.client.clean_params(context.params), 391 | ) 392 | context.response = completion.model_dump() 393 | return JSONResponse(content=context.response, status_code=200) 394 | except Exception as e: 395 | # Log the exception or handle it as needed 396 | print(e) 397 | context.response = { 398 | "error": "An error occurred processing the tool response" 399 | } 400 | return JSONResponse(content=context.response, status_code=500) 401 | 402 | return await super().handle(context) 403 | 404 | 405 | class DefaultCompletionHandler(Handler): 406 | async def handle(self, context: Context): 407 | if context.is_normal_chat: 408 | # Assuming context.client is set and has a method for creating chat completions 409 | completion = context.client.route( 410 | messages=context.messages, 411 | **context.client.clean_params(context.params), 412 | ) 413 | context.response = completion.model_dump() 414 | return JSONResponse(content=context.response, status_code=200) 415 | 416 | return await super().handle(context) 417 | 418 | 419 | class FallbackHandler(Handler): 420 | async def handle(self, context: Context): 421 | # This handler does not pass the request further down the chain. 422 | # It acts as a fallback when no other handler has processed the request. 423 | if not context.response: 424 | # The default action when no other handlers have processed the request 425 | context.response = {"message": "No suitable action found for the request."} 426 | return JSONResponse(content=context.response, status_code=400) 427 | 428 | # If there's already a response set in the context, it means one of the handlers has processed the request. 429 | return JSONResponse(content=context.response, status_code=200) 430 | 431 | 432 | class ExceptionHandler(Handler): 433 | async def handle(self, context: Context, exception: Exception): 434 | print(f"Error processing the request: {exception}") 435 | print(traceback.format_exc()) 436 | return JSONResponse( 437 | content={"error": "An unexpected error occurred. " + str(exception)}, 438 | status_code=500, 439 | ) 440 | -------------------------------------------------------------------------------- /app/libs/context.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict 2 | from fastapi import Request 3 | from providers import BaseProvider 4 | from prompts import * 5 | import importlib 6 | from utils import create_logger 7 | 8 | 9 | class Context: 10 | def __init__(self, request: Request, provider: str, body: Dict[str, Any]): 11 | self.request = request 12 | self.provider = provider 13 | self.body = body 14 | self.response = None 15 | 16 | # extract all keys from body except messages and tools and set in params 17 | self.params = {k: v for k, v in body.items() if k not in ["messages", "tools"]} 18 | 19 | # self.no_tool_behaviour = self.params.get("no_tool_behaviour", "return") 20 | self.no_tool_behaviour = self.params.get("no_tool_behaviour", "forward") 21 | self.params.pop("no_tool_behaviour", None) 22 | 23 | # Todo: For now, no stream, sorry ;) 24 | self.params["stream"] = False 25 | 26 | self.messages = body.get("messages", []) 27 | self.tools = body.get("tools", []) 28 | 29 | self.builtin_tools = [ 30 | t for t in self.tools if "parameters" not in t["function"] 31 | ] 32 | self.builtin_tool_names = [t["function"]["name"] for t in self.builtin_tools] 33 | self.custom_tools = [t for t in self.tools if "parameters" in t["function"]] 34 | 35 | for bt in self.builtin_tools: 36 | func_namespace = bt["function"]["name"] 37 | if len(func_namespace.split(".")) == 2: 38 | module_name, func_class_name = func_namespace.split(".") 39 | func_class_name = f"{func_class_name.capitalize()}Function" 40 | # raise ValueError("Only one builtin function can be called at a time.") 41 | module = importlib.import_module(f"app.functions.{module_name}") 42 | func_class = getattr(module, func_class_name, None) 43 | schema_dict = func_class.get_schema() 44 | if schema_dict: 45 | bt["function"] = schema_dict 46 | bt["run"] = func_class.run 47 | bt["extra"] = self.params.get("extra", {}) 48 | self.params.pop("extra", None) 49 | 50 | self.client: BaseProvider = None 51 | 52 | @property 53 | def last_message(self): 54 | return self.messages[-1] if self.messages else {} 55 | 56 | @property 57 | def is_tool_call(self): 58 | return bool( 59 | self.last_message["role"] == "user" 60 | and self.tools 61 | and self.params.get("tool_choice", None) != "none" 62 | ) 63 | 64 | @property 65 | def is_tool_response(self): 66 | return bool(self.last_message["role"] == "tool" and self.tools) 67 | 68 | @property 69 | def is_normal_chat(self): 70 | return bool(not self.is_tool_call and not self.is_tool_response) 71 | -------------------------------------------------------------------------------- /app/libs/provider_handler.py: -------------------------------------------------------------------------------- 1 | from importlib import import_module 2 | from fastapi.responses import JSONResponse 3 | from prompts import * 4 | from .base_handler import Handler 5 | from .context import Context 6 | 7 | class ProviderSelectionHandler(Handler): 8 | @staticmethod 9 | def provider_exists(provider: str) -> bool: 10 | module_name = f"app.providers" 11 | class_name = f"{provider.capitalize()}Provider" 12 | try: 13 | provider_module = import_module(module_name) 14 | provider_class = getattr(provider_module, class_name) 15 | return bool(provider_class) 16 | except ImportError: 17 | return False 18 | 19 | async def handle(self, context: Context): 20 | # Construct the module path and class name based on the provider 21 | module_name = f"app.providers" 22 | class_name = f"{context.provider.capitalize()}Provider" 23 | 24 | try: 25 | # Dynamically import the module and class 26 | provider_module = import_module(module_name) 27 | provider_class = getattr(provider_module, class_name) 28 | 29 | if provider_class: 30 | context.client = provider_class( 31 | api_key=context.api_token 32 | ) # Assuming an api_key parameter 33 | return await super().handle(context) 34 | else: 35 | raise ValueError( 36 | f"Provider class {class_name} could not be found in {module_name}." 37 | ) 38 | except ImportError as e: 39 | # Handle import error (e.g., module or class not found) 40 | print(f"Error importing {class_name} from {module_name}: {e}") 41 | context.response = { 42 | "error": f"An error occurred while trying to load the provider: {e}" 43 | } 44 | return JSONResponse(content=context.response, status_code=500) 45 | 46 | -------------------------------------------------------------------------------- /app/libs/tools_handler.py: -------------------------------------------------------------------------------- 1 | import concurrent.futures 2 | import uuid 3 | import json 4 | import math 5 | from fastapi.responses import JSONResponse 6 | from prompts import * 7 | from .base_handler import Handler, Context 8 | from .context import Context 9 | from utils import get_tool_call_response, create_logger, describe 10 | from config import PARSE_ERROR_TRIES, EVALUATION_CYCLES_COUNT 11 | from collections import defaultdict 12 | 13 | missed_tool_logger = create_logger( 14 | "chain.missed_tools", ".logs/empty_tool_tool_response.log" 15 | ) 16 | 17 | 18 | class ImageLLavaMessageHandler(Handler): 19 | async def handle(self, context: Context): 20 | new_messages = [] 21 | image_ref = 1 22 | for message in context.messages: 23 | new_messages.append(message) 24 | if message["role"] == "user": 25 | if isinstance(message["content"], list): 26 | for content in message["content"]: 27 | if content["type"] == "text": 28 | prompt = content["text"] 29 | elif content["type"] == "image_url": 30 | image_url = content["image_url"]["url"] 31 | try: 32 | description = describe(prompt, image_url) 33 | new_messages.append( 34 | {"role": "assistant", "content": description} 35 | ) 36 | image_ref += 1 37 | except Exception as e: 38 | print(f"Error describing image: {e}") 39 | continue 40 | context.messages = new_messages 41 | return await super().handle(context) 42 | 43 | 44 | class ToolExtractionHandler(Handler): 45 | async def handle(self, context: Context): 46 | if not context.is_tool_call: 47 | return await super().handle(context) 48 | 49 | # Step 1: Prepare the conversation history 50 | messages = self._prepare_conversation_history(context.messages) 51 | 52 | # Step 2: Prepare tool details and detect the mode of operation 53 | available_tools, system_message, suffix = self._prepare_tool_details(context) 54 | 55 | # Step 3: Prepare the messages for the model 56 | new_messages = self._prepare_model_messages(messages, available_tools, suffix, context.messages[-1]['content'], system_message) 57 | 58 | # Step 4: Detect the tool calls 59 | tool_calls_result = await self.process_tool_calls(context, new_messages) 60 | tool_calls = tool_calls_result["tool_calls"] 61 | 62 | # Step 5: Handle the situation where no tool calls are detected 63 | if not tool_calls: 64 | return await self._handle_no_tool_calls(context, tool_calls_result) 65 | 66 | # Step 6: Process built-in tools and resolve the tool calls 67 | unresolved_tool_calls, resolved_responses = self._process_builtin_tools(context, tool_calls, tool_calls_result["last_completion"].id) 68 | 69 | if not unresolved_tool_calls: 70 | context.messages.extend(resolved_responses) 71 | return await super().handle(context) 72 | 73 | # Step 7: Return the unresolved tool calls to the client 74 | tool_response = get_tool_call_response(tool_calls_result, unresolved_tool_calls, resolved_responses) 75 | context.response = tool_response 76 | return JSONResponse(content=context.response, status_code=200) 77 | 78 | def _prepare_conversation_history(self, messages): 79 | return [ 80 | f"<{m['role'].lower()}>\n{m['content']}\n" 81 | for m in messages 82 | if m["role"] != "system" 83 | ] 84 | 85 | def _prepare_tool_details(self, context): 86 | tool_choice = context.params.get("tool_choice", "auto") 87 | forced_mode = type(tool_choice) == dict and tool_choice.get("type", None) == "function" 88 | available_tools = [] 89 | 90 | if forced_mode: 91 | tool_choice = tool_choice["function"]["name"] 92 | available_tools = [t["function"] for t in context.tools if t["function"]["name"] == tool_choice] 93 | system_message = ENFORCED_SYSTAME_MESSAE 94 | suffix = get_forced_tool_suffix(tool_choice) 95 | else: 96 | tool_choice = "auto" 97 | available_tools = [t["function"] for t in context.tools] 98 | system_message = SYSTEM_MESSAGE 99 | suffix = get_suffix() 100 | 101 | # Add one special tool called "fallback", which is always available, its job is to be used when non of other tools are useful for the user input. 102 | # available_tools.append({ 103 | # "name": "fallback", 104 | # "description": "Use this tool when none of the other tools are useful for the user input.", 105 | # "arguments": {}} 106 | # ) 107 | 108 | return available_tools, system_message, suffix 109 | 110 | def _prepare_model_messages(self, messages, available_tools, suffix, last_message_content, system_message): 111 | messages_flatten = "\n".join(messages) 112 | tools_json = json.dumps(available_tools, indent=4) 113 | 114 | return [ 115 | {"role": "system", "content": system_message}, 116 | { 117 | "role": "user", 118 | "content": f"# Conversation History:\n{messages_flatten}\n\n# Available Tools: \n{tools_json}\n\n{suffix}\n{last_message_content}", 119 | }, 120 | ] 121 | 122 | async def _handle_no_tool_calls(self, context, tool_calls_result): 123 | if context.no_tool_behaviour == "forward": 124 | context.tools = None 125 | return await super().handle(context) 126 | else: 127 | context.response = {"tool_calls": []} 128 | tool_response = get_tool_call_response(tool_calls_result, [], []) 129 | missed_tool_logger.debug(f"Last message content: {context.last_message['content']}") 130 | return JSONResponse(content=tool_response, status_code=200) 131 | 132 | def _process_builtin_tools(self, context, tool_calls, tool_calls_result_id): 133 | unresolved_tool_calls = [ 134 | t 135 | for t in tool_calls 136 | if t["function"]["name"] not in context.builtin_tool_names 137 | ] 138 | resolved_responses = [] 139 | 140 | for tool in tool_calls: 141 | for bt in context.builtin_tools: 142 | if tool["function"]["name"] == bt["function"]["name"]: 143 | res = bt["run"](**{**json.loads(tool["function"]["arguments"]), **bt["extra"]}) 144 | resolved_responses.append({ 145 | "name": tool["function"]["name"], 146 | "role": "tool", 147 | "content": json.dumps(res), 148 | "tool_call_id": "chatcmpl-" + tool_calls_result_id, 149 | }) 150 | 151 | return unresolved_tool_calls, resolved_responses 152 | 153 | async def handle1(self, context: Context): 154 | body = context.body 155 | if context.is_tool_call: 156 | # Step 1: Prepare the the history of conversation. 157 | messages = [ 158 | f"<{m['role'].lower()}>\n{m['content']}\n" 159 | for m in context.messages 160 | if m["role"] != "system" 161 | ] 162 | messages_flatten = "\n".join(messages) 163 | 164 | 165 | # Step 2: Prepare tools details and detect the mode of operation. 166 | tool_choice = context.params.get("tool_choice", "auto") 167 | forced_mode = type(tool_choice) == dict and tool_choice.get("type", None) == "function" 168 | 169 | if forced_mode: 170 | tool_choice = tool_choice["function"]["name"] 171 | tools_json = json.dumps([t["function"] for t in context.tools if t["function"]["name"] == tool_choice], indent=4) 172 | system_message = ENFORCED_SYSTAME_MESSAE 173 | suffix = get_forced_tool_suffix(tool_choice) 174 | else: 175 | tool_choice = "auto" 176 | tools_json = json.dumps([t["function"] for t in context.tools], indent=4) 177 | system_message = SYSTEM_MESSAGE 178 | suffix = SUFFIX 179 | 180 | # Step 3: Prepare the messages for the model. 181 | new_messages = [ 182 | {"role": "system", "content": system_message}, 183 | { 184 | "role": "user", 185 | "content": f"# Conversation History:\n{messages_flatten}\n\n# Available Tools: \n{tools_json}\n\n{suffix}\n{context.messages[-1]['content']}", 186 | }, 187 | ] 188 | 189 | # Step 4: Detect the tool calls. 190 | tool_calls_result = await self.process_tool_calls(context, new_messages) 191 | tool_calls = tool_calls_result["tool_calls"] 192 | 193 | 194 | # Step 5: Handle the situation where no tool calls are detected. 195 | if not tool_calls: 196 | if context.no_tool_behaviour == "forward": 197 | context.tools = None 198 | return await super().handle(context) 199 | else: 200 | context.response = {"tool_calls": []} 201 | tool_response = get_tool_call_response(tool_calls_result, [], []) 202 | missed_tool_logger.debug( 203 | f"Last message content: {context.last_message['content']}" 204 | ) 205 | return JSONResponse(content=tool_response, status_code=200) 206 | 207 | 208 | # Step 6: Process built-in toola and resolve the tool calls, here on the server. In case there is unresolved tool calls, we will return the tool calls to the client to resolve them. But if all tool calls are resolved, we will continue to the next handler. 209 | unresolved_tol_calls = [ 210 | t 211 | for t in tool_calls 212 | if t["function"]["name"] not in context.builtin_tool_names 213 | ] 214 | resolved_responses = [] 215 | for tool in tool_calls: 216 | for bt in context.builtin_tools: 217 | if tool["function"]["name"] == bt["function"]["name"]: 218 | res = bt["run"]( 219 | **{ 220 | **json.loads(tool["function"]["arguments"]), 221 | **bt["extra"], 222 | } 223 | ) 224 | resolved_responses.append( 225 | { 226 | "name": tool["function"]["name"], 227 | "role": "tool", 228 | "content": json.dumps(res), 229 | "tool_call_id": "chatcmpl-" + tool_calls_result.id, 230 | } 231 | ) 232 | 233 | if not unresolved_tol_calls: 234 | context.messages.extend(resolved_responses) 235 | return await super().handle(context) 236 | 237 | # Step 7: If reach here, it means there are unresolved tool calls. We will return the tool calls to the client to resolve them. 238 | tool_response = get_tool_call_response( 239 | tool_calls_result, unresolved_tol_calls, resolved_responses 240 | ) 241 | 242 | context.response = tool_response 243 | return JSONResponse(content=context.response, status_code=200) 244 | 245 | return await super().handle(context) 246 | 247 | async def process_tool_calls(self, context, new_messages): 248 | try: 249 | evaluation_cycles_count = EVALUATION_CYCLES_COUNT 250 | 251 | def call_route(messages): 252 | completion = context.client.route( 253 | model=context.client.parser_model, 254 | messages=messages, 255 | temperature=0, 256 | max_tokens=512, 257 | top_p=1, 258 | stream=False, 259 | ) 260 | 261 | response = completion.choices[0].message.content 262 | response = response.replace("\_", "_") 263 | if TOOLS_OPEN_TOKEN in response: 264 | response = response.split(TOOLS_OPEN_TOKEN)[1].split( 265 | TOOLS_CLOSE_TOKEN 266 | )[0] 267 | if "```json" in response: 268 | response = response.split("```json")[1].split("```")[0] 269 | 270 | try: 271 | tool_response = json.loads(response) 272 | if isinstance(tool_response, list): 273 | tool_response = {"tool_calls": tool_response} 274 | # Check all detected functions exist in the available tools 275 | valid_names = [t['function']["name"] for t in context.tools] 276 | available_tools = [t for t in tool_response.get("tool_calls", []) if t['name'] in valid_names] 277 | tool_response = { 278 | "tool_calls": available_tools, 279 | } 280 | # tool_response = {"tool_calls": []} 281 | 282 | 283 | return tool_response.get("tool_calls", []), completion 284 | except json.JSONDecodeError as e: 285 | print(f"Error parsing the tool response: {e}") 286 | return [], None 287 | 288 | with concurrent.futures.ThreadPoolExecutor() as executor: 289 | futures = [ 290 | executor.submit(call_route, new_messages) 291 | for _ in range(evaluation_cycles_count) 292 | ] 293 | results = [ 294 | future.result() 295 | for future in concurrent.futures.as_completed(futures) 296 | ] 297 | 298 | tool_calls_list, completions = zip(*results) 299 | 300 | tool_calls_count = defaultdict(int) 301 | for tool_calls in tool_calls_list: 302 | for func in tool_calls: 303 | tool_calls_count[func["name"]] += 1 304 | 305 | pickup_threshold = math.floor(evaluation_cycles_count * 0.7) 306 | final_tool_calls = [] 307 | for tool_calls in tool_calls_list: 308 | for func in tool_calls: 309 | if tool_calls_count[func["name"]] >= pickup_threshold: 310 | # ppend if function is not already in the list 311 | if not any( 312 | f['function']["name"] == func["name"] for f in final_tool_calls 313 | ): 314 | final_tool_calls.append( 315 | { 316 | "id": f"call_{func['name']}_{str(uuid.uuid4())}", 317 | "type": "function", 318 | "function": { 319 | "name": func["name"], 320 | "arguments": json.dumps(func["arguments"]), 321 | }, 322 | } 323 | ) 324 | 325 | total_prompt_tokens = sum(c.usage.prompt_tokens for c in completions if c) 326 | total_completion_tokens = sum( 327 | c.usage.completion_tokens for c in completions if c 328 | ) 329 | total_tokens = sum(c.usage.total_tokens for c in completions if c) 330 | 331 | last_completion = completions[-1] if completions else None 332 | 333 | return { 334 | "tool_calls": final_tool_calls, 335 | "last_completion": last_completion, 336 | "usage": { 337 | "prompt_tokens": total_prompt_tokens, 338 | "completion_tokens": total_completion_tokens, 339 | "total_tokens": total_tokens, 340 | }, 341 | } 342 | 343 | except Exception as e: 344 | print(f"Error processing the tool calls: {e}") 345 | raise e 346 | 347 | 348 | class ToolResponseHandler(Handler): 349 | async def handle(self, context: Context): 350 | body = context.body 351 | if context.is_tool_response: 352 | messages = context.messages 353 | 354 | for message in messages: 355 | if message["role"] == "tool": 356 | message["role"] = "user" 357 | message["content"] = get_func_result_guide(message["content"]) 358 | 359 | try: 360 | params = { 361 | "temperature": 0.5, 362 | "max_tokens": 1024, 363 | } 364 | params = {**params, **context.params} 365 | 366 | completion = context.client.route( 367 | messages=messages, 368 | **context.client.clean_params(params), 369 | ) 370 | context.response = completion.model_dump() 371 | return JSONResponse(content=context.response, status_code=200) 372 | except Exception as e: 373 | raise e 374 | 375 | return await super().handle(context) 376 | -------------------------------------------------------------------------------- /app/libs/vision_handler.py: -------------------------------------------------------------------------------- 1 | from prompts import * 2 | from utils import describe 3 | from .context import Context 4 | from .base_handler import Handler 5 | 6 | 7 | class ImageMessageHandler(Handler): 8 | async def handle(self, context: Context): 9 | new_messages = [] 10 | image_ref = 1 11 | for message in context.messages: 12 | if message["role"] == "user": 13 | if isinstance(message["content"], list): 14 | prompt = None 15 | for content in message["content"]: 16 | if content["type"] == "text": 17 | # new_messages.append({"role": message["role"], "content": content["text"]}) 18 | prompt = content["text"] 19 | elif content["type"] == "image_url": 20 | image_url = content["image_url"]["url"] 21 | try: 22 | prompt = prompt or IMAGE_DESCRIPTO_PROMPT 23 | description = describe(prompt, image_url) 24 | if description: 25 | description = get_image_desc_guide(image_ref, description) 26 | new_messages.append( 27 | {"role": message["role"], "content": description} 28 | ) 29 | image_ref += 1 30 | else: 31 | pass 32 | except Exception as e: 33 | print(f"Error describing image: {e}") 34 | continue 35 | else: 36 | new_messages.append(message) 37 | else: 38 | new_messages.append(message) 39 | 40 | context.messages = new_messages 41 | return await super().handle(context) 42 | 43 | 44 | class ImageLLavaMessageHandler(Handler): 45 | async def handle(self, context: Context): 46 | new_messages = [] 47 | image_ref = 1 48 | for message in context.messages: 49 | new_messages.append(message) 50 | if message["role"] == "user": 51 | if isinstance(message["content"], list): 52 | for content in message["content"]: 53 | if content["type"] == "text": 54 | prompt = content["text"] 55 | elif content["type"] == "image_url": 56 | image_url = content["image_url"]["url"] 57 | try: 58 | description = describe(prompt, image_url) 59 | new_messages.append( 60 | {"role": "assistant", "content": description} 61 | ) 62 | image_ref += 1 63 | except Exception as e: 64 | print(f"Error describing image: {e}") 65 | continue 66 | context.messages = new_messages 67 | return await super().handle(context) 68 | 69 | -------------------------------------------------------------------------------- /app/main.py: -------------------------------------------------------------------------------- 1 | from fastapi import FastAPI 2 | from fastapi.responses import HTMLResponse 3 | from fastapi.templating import Jinja2Templates 4 | from fastapi.staticfiles import StaticFiles 5 | from starlette.middleware.cors import CORSMiddleware 6 | from starlette.requests import Request 7 | from routes import proxy 8 | from routes import examples 9 | from utils import create_logger 10 | import os 11 | from dotenv import load_dotenv 12 | 13 | load_dotenv() 14 | 15 | app = FastAPI() 16 | 17 | logger = create_logger("app", ".logs/access.log") 18 | app.mount("/static", StaticFiles(directory="frontend/assets"), name="static") 19 | templates = Jinja2Templates(directory="frontend/pages") 20 | 21 | 22 | origins = [ 23 | "*", 24 | ] 25 | 26 | app.add_middleware( 27 | CORSMiddleware, 28 | allow_origins=origins, 29 | allow_credentials=True, 30 | allow_methods=["*"], 31 | allow_headers=["*"], 32 | ) 33 | 34 | 35 | @app.middleware("http") 36 | async def log_requests(request: Request, call_next): 37 | if "/proxy" in request.url.path: 38 | client_ip = request.client.host 39 | logger.info( 40 | f"Incoming request from {client_ip}: {request.method} {request.url}" 41 | ) 42 | response = await call_next(request) 43 | # logger.info(f"Response status code: {response.status_code}") 44 | return response 45 | else: 46 | return await call_next(request) 47 | 48 | 49 | app.include_router(proxy.router, prefix="/proxy") 50 | app.include_router(examples.router, prefix="/examples") 51 | 52 | 53 | @app.get("/", response_class=HTMLResponse) 54 | async def index(request: Request): 55 | return templates.TemplateResponse("index.html", {"request": request}) 56 | 57 | 58 | # Add an get endpoint simple return the evrsion of the app 59 | @app.get("/version") 60 | async def version(): 61 | return {"version": "0.0.5"} 62 | 63 | 64 | if __name__ == "__main__": 65 | import uvicorn 66 | 67 | # uvicorn.run("main:app", host=os.getenv("HOST"), port=int(os.getenv('PORT')), workers=1, reload=True) 68 | uvicorn.run( 69 | "main:app", host=os.getenv("HOST"), port=int(os.getenv("PORT")), workers=1, reload=False 70 | ) 71 | -------------------------------------------------------------------------------- /app/models.py: -------------------------------------------------------------------------------- 1 | # To be developed -------------------------------------------------------------------------------- /app/providers.py: -------------------------------------------------------------------------------- 1 | from groq import Groq 2 | from openai import OpenAI 3 | from litellm import completion as sync_call_llm 4 | import litellm 5 | 6 | 7 | class BaseProvider: 8 | def __init__(self, api_key: str, base_url = None): 9 | self.api_key = api_key 10 | self.parser_model = "" 11 | self.route_model = "" 12 | 13 | def route(self, model: str, messages: list, **kwargs): 14 | pass 15 | 16 | async def route_async(self, model: str, messages: list, **kwargs): 17 | pass 18 | 19 | def clean_params(self, params): 20 | pass 21 | 22 | 23 | class OpenaiProvider(BaseProvider): 24 | def __init__(self, api_key: str, base_url = None): 25 | super().__init__(api_key) 26 | self._client = OpenAI(api_key=api_key) 27 | self.parser_model = "gpt-3.5-turbo" 28 | self.route_model = "gpt-3.5-turbo" 29 | self.exclude_params = ["messages"] 30 | 31 | def route(self, model: str, messages: list, **kwargs): 32 | completion = self._client.chat.completions.create( 33 | model=model, 34 | messages=messages, 35 | **kwargs 36 | ) 37 | return completion 38 | 39 | def clean_params(self, params): 40 | return {k: v for k, v in params.items() if k not in self.exclude_params} 41 | 42 | 43 | class GroqProvider(BaseProvider): 44 | def __init__(self, api_key: str, base_url = None): 45 | super().__init__(api_key) 46 | self._client = Groq(api_key=api_key) 47 | self.parser_model = "mixtral-8x7b-32768" 48 | self.route_model = "mixtral-8x7b-32768" 49 | self.exclude_params = ["messages", "tools", "tool_choice"] 50 | 51 | def route(self, model: str, messages: list, **kwargs): 52 | completion = self._client.chat.completions.create( 53 | model=model, 54 | messages=messages, 55 | **kwargs 56 | ) 57 | return completion 58 | 59 | def clean_params(self, params): 60 | return {k: v for k, v in params.items() if k not in self.exclude_params} 61 | 62 | 63 | class OllamaProvider(BaseProvider): 64 | def __init__(self, api_key: str, base_url = None): 65 | super().__init__(api_key) 66 | self.parser_model = "gemma:2b" 67 | self.route_model = "gemma:7b" 68 | self.exclude_params = ["messages", "tools", "tool_choice"] 69 | 70 | def route(self, model: str, messages: list, **kwargs): 71 | # Filter out all messages with rol assistant and has key "tool_calls" 72 | messages = [message for message in messages if message["role"] != "assistant" and "tool_calls" not in message] 73 | params = self.clean_params(kwargs) 74 | params = { 75 | 'max_tokens': 2048, 76 | **params 77 | } 78 | response = sync_call_llm( 79 | model=f"ollama/{model}", 80 | api_base="http://localhost:11434", 81 | messages=messages, 82 | **self.clean_params(kwargs) 83 | ) 84 | 85 | return response 86 | 87 | def clean_params(self, params): 88 | return {k: v for k, v in params.items() if k not in self.exclude_params} -------------------------------------------------------------------------------- /app/reasoning/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unclecode/groqcall/009c301a8a7125fba5a5e7eb0e0c0b1fba4aa211/app/reasoning/__init__.py -------------------------------------------------------------------------------- /app/reasoning/base.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from typing import Dict 3 | 4 | class ReasoningBase(ABC): 5 | name: str 6 | description: str 7 | 8 | @abstractmethod 9 | def run(self, context) -> Dict: 10 | pass 11 | 12 | -------------------------------------------------------------------------------- /app/reasoning/rerank.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel, Field 2 | from typing import Optional, Dict 3 | from .base import ReasoningBase 4 | from pydantic import Field 5 | from typing import Optional 6 | import requests 7 | import json, os 8 | from providers import GroqProvider 9 | import concurrent.futures 10 | from dotenv import load_dotenv 11 | load_dotenv() 12 | 13 | def get_rerank_prompt(query, responses, top_k): 14 | prompt = f"""You are an AI assistant tasked with evaluating and selecting the best responses to a user's request. The user's request is: 15 | 16 | 17 | {query} 18 | 19 | 20 | Here are the responses generated by different programmers to the user's request: 21 | 22 | {responses} 23 | 24 | # Task: 25 | Your task is to evaluate these responses and select the top {top_k} that best address the user's request. Consider factors such as relevance, clarity, and completeness when making your selection. 26 | 27 | After selecting the top {top_k} responses, generate a final response by merging and summarizing the selected responses. Format your output as follows: 28 | 29 | **Make sure to wrap the final answer supposed to be back to user using >>>** 30 | 31 | # Example of your response: 32 | After evaluating the responses, I have selected the top {top_k} that best address the user's request. Here they are: 33 | 34 | 35 | summary of response 1 36 | 37 | 38 | summary of response 2 39 | 40 | ... 41 | 42 | 43 | summary of response top_k 44 | 45 | Based on these top {top_k} responses, I have generated a final response by merging and summarizing them: 46 | 47 | >>> 48 | final_response_for_user 49 | >>>""" 50 | 51 | return prompt 52 | 53 | class RerankReasoning(ReasoningBase): 54 | name = "rerank" 55 | description = "Use this reasoning strategy to generate and rerank responses to a user query." 56 | 57 | def __init__(self, generator_model: str, reranker_model: str, n: int = 5, top_k: int = 3): 58 | self.generator_model = generator_model 59 | self.reranker_model = reranker_model 60 | self.n = n 61 | self.top_k = top_k 62 | self.generator_provider = GroqProvider(api_key=os.getenv("GROQ_API_KEY")) 63 | self.reranker_provider = GroqProvider(api_key=os.getenv("GROQ_API_KEY")) 64 | 65 | def run(self, context): 66 | message_stories = context.messages 67 | 68 | # Generate responses in parallel using a thread pool 69 | with concurrent.futures.ThreadPoolExecutor() as executor: 70 | response_futures = [executor.submit(self.generator_provider.route, model=self.generator_model, messages=message_stories) for _ in range(self.n)] 71 | responses = [future.result().get("response") for future in concurrent.futures.as_completed(response_futures)] 72 | 73 | unique_responses = list(set(responses)) 74 | 75 | # Adjust top_k if it's greater than the number of unique responses 76 | if self.top_k > len(unique_responses): 77 | self.top_k = int(len(unique_responses) * 0.4) 78 | 79 | # Generate prompt for reranking 80 | prompt = get_rerank_prompt( 81 | query=message_stories[-1].content, 82 | responses='\n\n'.join([f'\n{response}\n' for idx, response in enumerate(unique_responses)]), 83 | top_k=self.top_k 84 | ) 85 | 86 | # Rerank responses 87 | rerank_completion = self.reranker_provider.route(model=self.reranker_model, messages=[{"content": prompt}]) 88 | reranked_response = rerank_completion.get("response") 89 | 90 | # Extract the final response 91 | final_response = reranked_response.split(">>>")[1].split(">>>")[0].strip() 92 | 93 | # Add the final response as a new message to the context 94 | new_message = {"role": "assistant", "content":final_response} 95 | context.messages.append(new_message) 96 | 97 | return new_message -------------------------------------------------------------------------------- /app/routes/__init__.py: -------------------------------------------------------------------------------- 1 | from .proxy import router as proxy_router 2 | from .examples import router as examples_router 3 | 4 | __all__ = [ "proxy_router", "examples_router" ] 5 | -------------------------------------------------------------------------------- /app/routes/examples.py: -------------------------------------------------------------------------------- 1 | 2 | from fastapi.responses import FileResponse 3 | from fastapi import APIRouter 4 | import os 5 | 6 | router = APIRouter() 7 | 8 | # Add endpoint to downl;oad files in the ../examples folder 9 | @router.get("/{file_path}") 10 | async def read_examples(file_path: str): 11 | # get parent directory 12 | parent = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) 13 | file_path = f"{parent}/examples/{file_path}" 14 | if os.path.exists(file_path): 15 | return FileResponse(file_path) 16 | else: 17 | return {"error": "File not found."} 18 | 19 | # @router.get("/examples") 20 | # async def read_root(): 21 | # return {"message": "Hello World", "examples": [ 22 | # "/example/example_1.py", 23 | # "/example/example_2.py", 24 | # "/example/example_3.py", 25 | # "/example/example_4.py", 26 | # ]} -------------------------------------------------------------------------------- /app/routes/proxy.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter, Response, Request, Path, Query 2 | from fastapi.responses import JSONResponse 3 | # from libs.chains import ( 4 | # Context, 5 | # ProviderSelectionHandler, 6 | # ImageMessageHandler, 7 | # ToolExtractionHandler, 8 | # ToolResponseHandler, 9 | # DefaultCompletionHandler, 10 | # FallbackHandler, 11 | # ) 12 | 13 | from libs import ( 14 | Context, 15 | ProviderSelectionHandler, 16 | ImageMessageHandler, 17 | ToolExtractionHandler, 18 | ToolResponseHandler, 19 | DefaultCompletionHandler, 20 | FallbackHandler, 21 | ) 22 | 23 | 24 | from typing import Optional 25 | 26 | router = APIRouter() 27 | 28 | 29 | # Add get endpoint for /openai/v1 and print request body 30 | @router.get("/{provider}/v1") 31 | async def get_openai_v1( 32 | response: Response, provider: str = Path(..., title="Provider") 33 | ) -> JSONResponse: 34 | return JSONResponse(content={"message": f"GET request to {provider} v1"}) 35 | 36 | 37 | @router.post("/groqchain/{provider}/v1/chat/completions") 38 | async def post_groq_chat_completions( 39 | request: Request, 40 | provider: str = Path(..., title="Provider") 41 | ) -> JSONResponse: 42 | # Call the original post_chat_completions method with provider set to "groq" 43 | return await post_chat_completions(request, provider="groq") 44 | 45 | 46 | @router.post("/{provider}/v1/chat/completions") 47 | async def post_chat_completions( 48 | request: Request, 49 | provider: str = Path(..., title="Provider") 50 | ) -> JSONResponse: 51 | try: 52 | if not provider: 53 | provider = "openai" 54 | 55 | if not ProviderSelectionHandler.provider_exists(provider): 56 | return JSONResponse(content={"error": "Invalid provider"}, status_code=400) 57 | 58 | # Extract the API token and body from the request 59 | api_token = request.headers.get("Authorization").split("Bearer ")[1] 60 | body = await request.json() 61 | 62 | # Initialize the context with request details 63 | context = Context(request, provider, body) 64 | context.api_token = ( 65 | api_token # Adding the API token to the context for use in handlers 66 | ) 67 | 68 | # Initialize and link the handlers 69 | provider_selection_handler = ProviderSelectionHandler() 70 | image_message_handler = ImageMessageHandler() 71 | tool_extraction_handler = ToolExtractionHandler() 72 | tool_response_handler = ToolResponseHandler() 73 | default_completion_handler = DefaultCompletionHandler() 74 | fallback_handler = FallbackHandler() 75 | 76 | # Set up the chain of responsibility 77 | chains = [ 78 | provider_selection_handler, 79 | image_message_handler, 80 | tool_extraction_handler, 81 | tool_response_handler, 82 | default_completion_handler, 83 | fallback_handler, 84 | ] 85 | for i in range(len(chains) - 1): 86 | chains[i].set_next(chains[i + 1]) 87 | 88 | # provider_selection_handler.set_next(tool_extraction_handler).set_next( 89 | # tool_response_handler 90 | # ).set_next(default_completion_handler).set_next(fallback_handler) 91 | 92 | # Execute the chain with the initial context 93 | response = await provider_selection_handler.handle(context) 94 | 95 | # Return the response generated by the handlers 96 | return response 97 | except Exception as e: 98 | print(f"Error processing the request: {e}") 99 | return JSONResponse( 100 | content={"error": "An unexpected error occurred"}, status_code=500 101 | ) 102 | -------------------------------------------------------------------------------- /app/utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import replicate 4 | import base64 5 | from io import BytesIO 6 | 7 | 8 | # To be developed 9 | def create_logger(logger_name: str, log_path: str = ".logs/access.log", show_on_shell: bool = False): 10 | log_dir = os.path.dirname(log_path) 11 | if not os.path.exists(log_dir): 12 | os.makedirs(log_dir) 13 | logger = logging.getLogger(logger_name) 14 | logger.setLevel(logging.DEBUG) 15 | file_handler = logging.FileHandler(log_path) 16 | file_handler.setLevel(logging.DEBUG) 17 | formatter = logging.Formatter( 18 | "%(asctime)s - %(name)s - %(levelname)s - %(message)s" 19 | ) 20 | file_handler.setFormatter(formatter) 21 | logger.addHandler(file_handler) 22 | if show_on_shell: 23 | stream_handler = logging.StreamHandler() 24 | stream_handler.setLevel(logging.DEBUG) 25 | shell_formatter = logging.Formatter( 26 | "%(levelname)s (%(name)s) %(message)s" 27 | ) 28 | stream_handler.setFormatter(shell_formatter) 29 | logger.addHandler(stream_handler) 30 | return logger 31 | 32 | 33 | def get_tool_call_response(tool_calls_result, unresolved_tol_calls, resolved_responses): 34 | last_completion = tool_calls_result["last_completion"] 35 | tool_response = { 36 | "id": "chatcmpl-" + last_completion.id if last_completion else None, 37 | "object": "chat.completion", 38 | "created": last_completion.created if last_completion else None, 39 | "model": last_completion.model if last_completion else None, 40 | "choices": [ 41 | { 42 | "index": 0, 43 | "message": { 44 | "role": "assistant", 45 | "content": "", # None, 46 | "tool_calls": unresolved_tol_calls, 47 | }, 48 | "logprobs": None, 49 | "finish_reason": "tool_calls", 50 | } 51 | ], 52 | "resolved": resolved_responses, 53 | "usage": tool_calls_result["usage"], 54 | "system_fingerprint": last_completion.system_fingerprint if last_completion else None, 55 | } 56 | return tool_response 57 | 58 | def describe(prompt: str, image_url_or_base64 : str, **kwargs) -> str: 59 | logger = create_logger("vision", ".logs/access.log", True) 60 | try: 61 | if image_url_or_base64.startswith("data:image/"): 62 | # If the input is a base64 string 63 | image_data = base64.b64decode(image_url_or_base64.split(",")[1]) 64 | image_file = BytesIO(image_data) 65 | else: 66 | # If the input is a URL 67 | image_file = image_url_or_base64 68 | 69 | model_params = { 70 | "top_p": 1, 71 | "max_tokens": 1024, 72 | "temperature": 0.2 73 | } 74 | model_params.update(kwargs) 75 | 76 | logger.info("Running the model") 77 | output = replicate.run( 78 | "yorickvp/llava-13b:01359160a4cff57c6b7d4dc625d0019d390c7c46f553714069f114b392f4a726", 79 | input={ 80 | "image": image_file, 81 | "prompt": prompt, #"Describe the image in detail.", 82 | **model_params 83 | } 84 | ) 85 | 86 | description = "" 87 | for item in output: 88 | if not description: 89 | logger.info("Streaming...") 90 | description += item 91 | 92 | return description.strip() 93 | except Exception as e: 94 | logger.error( f"Vision model, An error occurred: {e}") 95 | return None 96 | 97 | 98 | 99 | # describe("Describe the image in detail.", "https://replicate.delivery/pbxt/KRULC43USWlEx4ZNkXltJqvYaHpEx2uJ4IyUQPRPwYb8SzPf/view.jpg") -------------------------------------------------------------------------------- /cookbook/ai_assistant_custome_tools.py: -------------------------------------------------------------------------------- 1 | 2 | import os, json 3 | from typing import Optional, List 4 | from phi.llm.openai.like import OpenAILike 5 | from phi.assistant import Assistant 6 | from phi.knowledge.json import JSONKnowledgeBase 7 | from phi.vectordb.pgvector import PgVector2 8 | from phi.storage.assistant.postgres import PgAssistantStorage 9 | from phi.tools import Toolkit 10 | from phi.tools.email import EmailTools 11 | from phi.utils.log import logger 12 | from phi.tools.email import EmailTools 13 | from phi.knowledge.base import AssistantKnowledge 14 | from phi.knowledge.base import Document 15 | from resources import vector_db 16 | from rich.prompt import Prompt 17 | from dotenv import load_dotenv 18 | load_dotenv() 19 | 20 | # To run this example, first make sure to follow the instructions below: 21 | # 1. Install the phidata: pip install phidata 22 | # 2. Run the following command to start a docker, with pgvector db running: phi start resources.py 23 | # 3. Download the sample of JSON knowledge base from the same folder of this file: cinemax.json 24 | 25 | class CinemaSerachDB(Toolkit): 26 | def __init__( 27 | self, 28 | knowledge_base : Optional[AssistantKnowledge] = None, 29 | num_documents: int = None 30 | ): 31 | super().__init__(name="get_available_slots") 32 | self.knowledge_base = knowledge_base 33 | self.num_documents = num_documents 34 | self.register(self.get_available_slots) 35 | 36 | def get_available_slots(self, movie_slot_query: str ) -> str: 37 | """Use this function to search the Cinemax database of available movies, show time, and date. 38 | 39 | :param query: The query to search the Cinemax database of available movies, show time, and date. 40 | :return: A string containing the response to the query. 41 | """ 42 | relevant_docs: List[Document] = self.knowledge_base.search(query=movie_slot_query, num_documents=self.num_documents) 43 | if len(relevant_docs) == 0: 44 | return None 45 | 46 | return json.dumps([doc.to_dict() for doc in relevant_docs], indent=2) 47 | 48 | 49 | 50 | class CinemaTools(Toolkit): 51 | def __init__( 52 | self, 53 | email_tools: Optional["EmailTools"] = None, 54 | ): 55 | super().__init__(name="cinema_tools") 56 | self.email_tools = email_tools 57 | self.register(self.book_cinema_ticket) 58 | 59 | def book_cinema_ticket(self, movie_name: str, date: Optional[str] = None, time: Optional[str] = None, user_email: Optional[str] = None) -> str: 60 | """Use this function ONLY for booking a ticket, when all info is available (movie name, date, time and suer email). Do NOT use this function when user asks for movie details and other things 61 | 62 | Args: 63 | movie_name (str): The name of the movie. 64 | date (Optional[str], optional): The date of the movie. 65 | time (Optional[str], optional): The time of the movie. 66 | user_email (Optional[str], optional): The email of the user. Defaults to None. 67 | 68 | Returns: 69 | The result of the operation. 70 | 71 | """ 72 | 73 | anything_missed = any([not movie_name, not date, not time, not user_email]) 74 | 75 | missed_items = [] 76 | 77 | if anything_missed: 78 | if not date: 79 | missed_items.append( "error: No date provided, I need a date to book a ticket") 80 | 81 | if not time: 82 | missed_items.append( "error: No time provided, I need a time to book a ticket") 83 | 84 | if not user_email: 85 | missed_items.append( "error: No user email provided, I need an email to send the ticket") 86 | 87 | missed_itemes = ", ".join(missed_items) 88 | return f"There are some missing items: \n{missed_itemes}" 89 | 90 | # Simulate booking the ticket 91 | ticket_number = self._generate_ticket_number() 92 | logger.info(f"Booking ticket for {movie_name} on {date} at {time}") 93 | 94 | # Prepare the email subject and body 95 | subject = f"Your ticket for {movie_name}" 96 | body = f"Dear user,\n\nYour ticket for {movie_name} on {date} at {time} has been booked.\n\n" \ 97 | f"Your ticket number is: {ticket_number}\n\nEnjoy the movie!\n\nBest regards,\nThe Cinema Team" 98 | 99 | # Send the email using the EmailTools 100 | if not self.email_tools: 101 | return "error: No email tools provided" 102 | self.email_tools.receiver_email = user_email 103 | result = self.email_tools.email_user(subject, body) 104 | 105 | if result.startswith("error"): 106 | logger.error(f"Error booking ticket: {result}") 107 | return result 108 | return "success" 109 | 110 | def _generate_ticket_number(self) -> str: 111 | """Generates a dummy ticket number.""" 112 | import random 113 | import string 114 | return "".join(random.choices(string.ascii_uppercase + string.digits, k=10)) 115 | 116 | kb = JSONKnowledgeBase( 117 | path="cinemax.json", 118 | vector_db=PgVector2(collection="cinemax", db_url=vector_db.get_db_connection_local()), 119 | ) 120 | storage = PgAssistantStorage( 121 | table_name="cinemax_assistant_storage", 122 | db_url=vector_db.get_db_connection_local(), 123 | ) 124 | 125 | my_groq = OpenAILike( 126 | model="mixtral-8x7b-32768", 127 | api_key=os.environ["GROQ_API_KEY"], 128 | base_url="http://localhost:8000/proxy/groq/v1" 129 | # base_url="http://groqcall.ai/proxy/groq/v1" 130 | ) 131 | 132 | 133 | def cinemax_assistant(new: bool = False, user: str = "user"): 134 | run_id: Optional[str] = None 135 | # new = False 136 | # new = True 137 | user_id = user 138 | 139 | if not new: 140 | existing_run_ids: List[str] = storage.get_all_run_ids(user_id) 141 | if len(existing_run_ids) > 0: 142 | run_id = existing_run_ids[0] 143 | 144 | assistant = Assistant( 145 | run_id=run_id, 146 | user_id="test_user", 147 | llm=my_groq, 148 | # knowledge_base=kb, Since I created my own tool CinemaSerachDB 149 | storage=storage, 150 | use_tools=True, 151 | tools=[ 152 | CinemaSerachDB(knowledge_base=kb, num_documents=5), 153 | CinemaTools(EmailTools("YOUR_EMAIL_ADDRESS", "SENDER_NAME", "SENDER_EMAIL", os.environ['email_pass_key'] )) 154 | ], show_tool_calls=True, markdown=True, 155 | add_chat_history_to_messages=True, # Inject history in every prompt 156 | add_datetime_to_instructions=True, 157 | limit_tool_access=True, 158 | extra_instructions="", 159 | output_model= None, # Pydantic 160 | prevent_prompt_injection=True, 161 | add_references_to_prompt=False, # It searches in the kb and add it to every prompt 162 | # add_references_to_prompt=True, 163 | add_to_system_prompt="\n\nRemember:As a Cinemax agent, your responsibilities include: \n\n1. Assisting individuals in locating their preferred movies using the provided database information. \n2. Facilitating ticket booking through the tools at your disposal. \n3. If critical information is missing and the registration system returns an error, generate a polite, humanized version of the error message and request the necessary additional information from users. \n4. When interacting with users, never disclose that you are an AI model unable to perform specific functions or activate certain features. \n5. When users inquire about movie showtimes, available movies, or related details, utilize your tools to access data from the knowledge base, rather than relying on your own knowledge. However, you may provide plot information about specific movies in a concise and succinct manner, ensuring responses are not excessively lengthy. \n6.When engaging in a discussion with users about films and proposing movies based on their preferences and the available options, it would be beneficial to list the available dates and times tailored to the user's interests and conversation history. This approach would simplify the booking process for the user. If the user has already specified a date, it is essential to remember and adhere to it, avoiding the suggestion of alternative dates." 164 | ) 165 | assistant.knowledge_base.load(recreate=False) 166 | 167 | if run_id is None: 168 | run_id = assistant.run_id 169 | print(f"Started Run: {run_id}\n") 170 | else: 171 | print(f"Continuing Run: {run_id}\n") 172 | 173 | while True: 174 | message = Prompt.ask(f"[bold] :sunglasses: {user} [/bold]") 175 | if message in ("exit", "bye"): 176 | break 177 | assistant.print_response(message, markdown=True, stream=False) 178 | # response = assistant.run(message, stream=False) 179 | 180 | if __name__ == "__main__": 181 | cinemax_assistant(user="Tom") 182 | 183 | 184 | -------------------------------------------------------------------------------- /cookbook/function_call_force_schema.py: -------------------------------------------------------------------------------- 1 | 2 | from duckduckgo_search import DDGS 3 | import requests, os 4 | import json 5 | 6 | api_key=os.environ["GROQ_API_KEY"] 7 | header = { 8 | "Authorization": f"Bearer {api_key}", 9 | "Content-Type": "application/json" 10 | } 11 | proxy_url = "https://groqcall.ai/proxy/groq/v1/chat/completions" 12 | 13 | # or "http://localhost:8000/proxy/groq/v1/chat/completions" if running locally 14 | # proxy_url = "http://localhost:8000/proxy/groq/v1/chat/completions" 15 | 16 | 17 | def duckduckgo_search(query, max_results=None): 18 | """ 19 | Use this function to search DuckDuckGo for a query. 20 | """ 21 | with DDGS() as ddgs: 22 | return [r for r in ddgs.text(query, safesearch='off', max_results=max_results)] 23 | 24 | def duckduckgo_news(query, max_results=None): 25 | """ 26 | Use this function to get the latest news from DuckDuckGo. 27 | """ 28 | with DDGS() as ddgs: 29 | return [r for r in ddgs.news(query, safesearch='off', max_results=max_results)] 30 | 31 | function_map = { 32 | "duckduckgo_search": duckduckgo_search, 33 | "duckduckgo_news": duckduckgo_news, 34 | } 35 | 36 | request = { 37 | "messages": [ 38 | { 39 | "role": "system", 40 | "content": "YOU MUST FOLLOW THESE INSTRUCTIONS CAREFULLY.\n\n1. Use markdown to format your answers.\n" 41 | }, 42 | { 43 | "role": "user", 44 | "content": "Whats happening in France? Summarize top stories with sources, very short and concise." 45 | } 46 | ], 47 | "model": "mixtral-8x7b-32768", 48 | # "tool_choice": "auto", 49 | # "tool_choice": "none", 50 | "tool_choice": {"type": "function", "function": {"name": "duckduckgo_search"}}, 51 | "tools": [ 52 | { 53 | "type": "function", 54 | "function": { 55 | "name": "duckduckgo_search", 56 | "description": "Use this function to search DuckDuckGo for a query.\n\nArgs:\n query(str): The query to search for.\n max_results (optional, default=5): The maximum number of results to return.\n\nReturns:\n The result from DuckDuckGo.", 57 | "parameters": { 58 | "type": "object", 59 | "properties": { 60 | "query": { 61 | "type": "string" 62 | }, 63 | "max_results": { 64 | "type": [ 65 | "number", 66 | "null" 67 | ] 68 | } 69 | } 70 | } 71 | } 72 | }, 73 | { 74 | "type": "function", 75 | "function": { 76 | "name": "duckduckgo_news", 77 | "description": "Use this function to get the latest news from DuckDuckGo.\n\nArgs:\n query(str): The query to search for.\n max_results (optional, default=5): The maximum number of results to return.\n\nReturns:\n The latest news from DuckDuckGo.", 78 | "parameters": { 79 | "type": "object", 80 | "properties": { 81 | "query": { 82 | "type": "string" 83 | }, 84 | "max_results": { 85 | "type": [ 86 | "number", 87 | "null" 88 | ] 89 | } 90 | } 91 | } 92 | } 93 | } 94 | ] 95 | } 96 | 97 | response = requests.post( 98 | proxy_url, 99 | headers=header, 100 | json=request 101 | ) 102 | # Check if the request was successful 103 | if response.status_code == 200: 104 | # Process the response data (if needed) 105 | res = response.json() 106 | message = res['choices'][0]['message'] 107 | tools_response_messages = [] 108 | if not message['content'] and 'tool_calls' in message: 109 | for tool_call in message['tool_calls']: 110 | tool_name = tool_call['function']['name'] 111 | tool_args = tool_call['function']['arguments'] 112 | tool_args = json.loads(tool_args) 113 | if tool_name not in function_map: 114 | print(f"Error: {tool_name} is not a valid function name.") 115 | continue 116 | tool_func = function_map[tool_name] 117 | tool_response = tool_func(**tool_args) 118 | tools_response_messages.append({ 119 | "role": "tool", "content": json.dumps(tool_response) 120 | }) 121 | 122 | if tools_response_messages: 123 | request['messages'] += tools_response_messages 124 | response = requests.post( 125 | proxy_url, 126 | headers=header, 127 | json=request 128 | ) 129 | if response.status_code == 200: 130 | res = response.json() 131 | print(res['choices'][0]['message']['content']) 132 | else: 133 | print("Error:", response.status_code, response.text) 134 | else: 135 | print(message['content']) 136 | else: 137 | print("Error:", response.status_code, response.text) 138 | -------------------------------------------------------------------------------- /cookbook/function_call_force_tool_choice.py: -------------------------------------------------------------------------------- 1 | from duckduckgo_search import DDGS 2 | import requests, os 3 | import json 4 | 5 | api_key=os.environ["GROQ_API_KEY"] 6 | header = { 7 | "Authorization": f"Bearer {api_key}", 8 | "Content-Type": "application/json" 9 | } 10 | 11 | # proxy_url = "https://funckycall.ai/proxy/groq/v1/chat/completions" 12 | proxy_url = "http://localhost:8000/proxy/groq/v1/chat/completions" 13 | 14 | def duckduckgo_search(query, max_results=None): 15 | """ 16 | Use this function to search DuckDuckGo for a query. 17 | """ 18 | with DDGS() as ddgs: 19 | return [r for r in ddgs.text(query, safesearch='off', max_results=max_results)] 20 | 21 | def duckduckgo_news(query, max_results=None): 22 | """ 23 | Use this function to get the latest news from DuckDuckGo. 24 | """ 25 | with DDGS() as ddgs: 26 | return [r for r in ddgs.news(query, safesearch='off', max_results=max_results)] 27 | 28 | function_map = { 29 | "duckduckgo_search": duckduckgo_search, 30 | "duckduckgo_news": duckduckgo_news, 31 | } 32 | 33 | 34 | request = { 35 | "messages": [ 36 | { 37 | "role": "user", 38 | "content": "Whats happening in France? Summarize top stories with sources, search in general and also search news, very short and concise.", 39 | } 40 | ], 41 | "model": "mixtral-8x7b-32768", 42 | # "tool_choice": "auto", 43 | # "tool_choice": None, 44 | "tool_choice": {"type": "function", "function": {"name": "duckduckgo_search"}}, 45 | "tools": [ 46 | { 47 | "type": "function", 48 | "function": { 49 | "name": "duckduckgo_search", 50 | "description": "Use this function to search DuckDuckGo for a query.\n\nArgs:\n query(str): The query to search for.\n max_results (optional, default=5): The maximum number of results to return.\n\nReturns:\n The result from DuckDuckGo.", 51 | "parameters": { 52 | "type": "object", 53 | "properties": { 54 | "query": {"type": "string"}, 55 | "max_results": {"type": ["number", "null"]}, 56 | }, 57 | }, 58 | }, 59 | }, 60 | { 61 | "type": "function", 62 | "function": { 63 | "name": "duckduckgo_news", 64 | "description": "Use this function to get the latest news from DuckDuckGo.\n\nArgs:\n query(str): The query to search for.\n max_results (optional, default=5): The maximum number of results to return.\n\nReturns:\n The latest news from DuckDuckGo.", 65 | "parameters": { 66 | "type": "object", 67 | "properties": { 68 | "query": {"type": "string"}, 69 | "max_results": {"type": ["number", "null"]}, 70 | }, 71 | }, 72 | }, 73 | }, 74 | ], 75 | } 76 | 77 | response = requests.post( 78 | proxy_url, 79 | headers=header, 80 | json=request, 81 | ) 82 | 83 | 84 | # Check if the request was successful 85 | if response.status_code == 200: 86 | # Process the response data (if needed) 87 | res = response.json() 88 | message = res['choices'][0]['message'] 89 | tools_response_messages = [] 90 | if not message['content'] and 'tool_calls' in message: 91 | for tool_call in message['tool_calls']: 92 | tool_name = tool_call['function']['name'] 93 | tool_args = tool_call['function']['arguments'] 94 | tool_args = json.loads(tool_args) 95 | if tool_name not in function_map: 96 | print(f"Error: {tool_name} is not a valid function name.") 97 | continue 98 | tool_func = function_map[tool_name] 99 | tool_response = tool_func(**tool_args) 100 | tools_response_messages.append({ 101 | "role": "tool", "content": json.dumps(tool_response) 102 | }) 103 | 104 | if tools_response_messages: 105 | request['messages'] += tools_response_messages 106 | response = requests.post( 107 | proxy_url, 108 | headers=header, 109 | json=request 110 | ) 111 | if response.status_code == 200: 112 | res = response.json() 113 | print(res['choices'][0]['message']['content']) 114 | else: 115 | print("Error:", response.status_code, response.text) 116 | else: 117 | print(message['content']) 118 | else: 119 | print("Error:", response.status_code, response.text) 120 | -------------------------------------------------------------------------------- /cookbook/function_call_ollama.py: -------------------------------------------------------------------------------- 1 | 2 | from phi.llm.openai.like import OpenAILike 3 | from phi.assistant import Assistant 4 | from phi.tools.duckduckgo import DuckDuckGo 5 | 6 | # Tried the proxy with Ollama and it works great, meaning we can use it with any provider. But, you never get the speed of Groq ;) 7 | my_ollama = OpenAILike( 8 | model="gemma:7b", 9 | api_key="", 10 | base_url="http://localhost:11235/proxy/ollama/v1" 11 | ) 12 | ollama_assistant = Assistant( 13 | llm=my_ollama, 14 | tools=[DuckDuckGo()], show_tool_calls=True, markdown=True 15 | ) 16 | ollama_assistant.print_response("Whats happening in France? Summarize top stories with sources, very short and concise.", stream=False) -------------------------------------------------------------------------------- /cookbook/function_call_phidata.py: -------------------------------------------------------------------------------- 1 | 2 | from phi.llm.openai.like import OpenAILike 3 | from phi.assistant import Assistant 4 | from phi.tools.duckduckgo import DuckDuckGo 5 | import os, json 6 | 7 | 8 | groq = OpenAILike( 9 | model="mixtral-8x7b-32768", 10 | api_key=os.environ["GROQ_API_KEY"], 11 | base_url="https://api.groq.com/openai/v1" 12 | ) 13 | assistant = Assistant( 14 | llm=groq, 15 | tools=[DuckDuckGo()], show_tool_calls=True, markdown=True 16 | ) 17 | 18 | # If you run without a proxy, you will get a error, becuase Groq does not have a function to call 19 | # assistant.print_response("Whats happening in France? Summarize top stories with sources, very short and concise.", stream=False) 20 | 21 | my_groq = OpenAILike( 22 | model="mixtral-8x7b-32768", # or model="gemma-7b-it", 23 | api_key=os.environ["GROQ_API_KEY"], 24 | base_url="https://groqcall.ai/proxy/groq/v1" # or "http://localhost:8000/proxy/groq/v1" if running locally 25 | ) 26 | assistant = Assistant( 27 | llm=my_groq, 28 | tools=[DuckDuckGo()], show_tool_calls=True, markdown=True 29 | ) 30 | assistant.print_response("Whats happening in France? Summarize top stories with sources, very short and concise.", stream=False) 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /cookbook/function_call_vision.py: -------------------------------------------------------------------------------- 1 | import requests, os 2 | 3 | api_key = os.environ["GROQ_API_KEY"] 4 | header = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"} 5 | 6 | proxy_url = "https://groqcall.ai/proxy/groq/v1/chat/completions" # or "http://localhost:8000/proxy/groq/v1/chat/completions" if running locally 7 | proxy_url = "http://localhost:8000/proxy/groq/v1/chat/completions" 8 | 9 | request = { 10 | "messages": [ 11 | { 12 | "role": "system", 13 | "content": "YOU MUST FOLLOW THESE INSTRUCTIONS CAREFULLY.\n\n1. Use markdown to format your answers.\n", 14 | }, 15 | { 16 | "role": "user", 17 | "content": [ 18 | {"type": "text", "text": "What’s in this image?"}, 19 | { 20 | "type": "image_url", 21 | "image_url": { 22 | "url": "https://res.cloudinary.com/kidocode/image/upload/v1710690498/Gfp-wisconsin-madison-the-nature-boardwalk_m9jalr.jpg" 23 | }, 24 | }, 25 | ], 26 | }, 27 | { 28 | "role": "user", 29 | # "content": "What’s in this image?", 30 | "content": "Generate 3 keywords for the image description", 31 | }, 32 | ], 33 | "model": "mixtral-8x7b-32768" 34 | } 35 | 36 | response = requests.post(proxy_url, headers=header, json=request) 37 | 38 | 39 | response.text 40 | 41 | print(response.json()["choices"][0]["message"]["content"]) 42 | -------------------------------------------------------------------------------- /cookbook/function_call_with_schema.py: -------------------------------------------------------------------------------- 1 | 2 | from duckduckgo_search import DDGS 3 | import requests, os 4 | import json 5 | 6 | api_key=os.environ["GROQ_API_KEY"] 7 | header = { 8 | "Authorization": f"Bearer {api_key}", 9 | "Content-Type": "application/json" 10 | } 11 | proxy_url = "https://groqcall.ai/proxy/groq/v1/chat/completions" # or "http://localhost:8000/proxy/groq/v1/chat/completions" if running locally 12 | 13 | 14 | def duckduckgo_search(query, max_results=None): 15 | """ 16 | Use this function to search DuckDuckGo for a query. 17 | """ 18 | with DDGS() as ddgs: 19 | return [r for r in ddgs.text(query, safesearch='off', max_results=max_results)] 20 | 21 | def duckduckgo_news(query, max_results=None): 22 | """ 23 | Use this function to get the latest news from DuckDuckGo. 24 | """ 25 | with DDGS() as ddgs: 26 | return [r for r in ddgs.news(query, safesearch='off', max_results=max_results)] 27 | 28 | function_map = { 29 | "duckduckgo_search": duckduckgo_search, 30 | "duckduckgo_news": duckduckgo_news, 31 | } 32 | 33 | request = { 34 | "messages": [ 35 | { 36 | "role": "system", 37 | "content": "YOU MUST FOLLOW THESE INSTRUCTIONS CAREFULLY.\n\n1. Use markdown to format your answers.\n" 38 | }, 39 | { 40 | "role": "user", 41 | "content": "Whats happening in France? Summarize top stories with sources, very short and concise." 42 | } 43 | ], 44 | "model": "mixtral-8x7b-32768", 45 | "tool_choice": "auto", 46 | "tools": [ 47 | { 48 | "type": "function", 49 | "function": { 50 | "name": "duckduckgo_search", 51 | "description": "Use this function to search DuckDuckGo for a query.\n\nArgs:\n query(str): The query to search for.\n max_results (optional, default=5): The maximum number of results to return.\n\nReturns:\n The result from DuckDuckGo.", 52 | "parameters": { 53 | "type": "object", 54 | "properties": { 55 | "query": { 56 | "type": "string" 57 | }, 58 | "max_results": { 59 | "type": [ 60 | "number", 61 | "null" 62 | ] 63 | } 64 | } 65 | } 66 | } 67 | }, 68 | { 69 | "type": "function", 70 | "function": { 71 | "name": "duckduckgo_news", 72 | "description": "Use this function to get the latest news from DuckDuckGo.\n\nArgs:\n query(str): The query to search for.\n max_results (optional, default=5): The maximum number of results to return.\n\nReturns:\n The latest news from DuckDuckGo.", 73 | "parameters": { 74 | "type": "object", 75 | "properties": { 76 | "query": { 77 | "type": "string" 78 | }, 79 | "max_results": { 80 | "type": [ 81 | "number", 82 | "null" 83 | ] 84 | } 85 | } 86 | } 87 | } 88 | } 89 | ] 90 | } 91 | 92 | response = requests.post( 93 | proxy_url, 94 | headers=header, 95 | json=request 96 | ) 97 | # Check if the request was successful 98 | if response.status_code == 200: 99 | # Process the response data (if needed) 100 | res = response.json() 101 | message = res['choices'][0]['message'] 102 | tools_response_messages = [] 103 | if not message['content'] and 'tool_calls' in message: 104 | for tool_call in message['tool_calls']: 105 | tool_name = tool_call['function']['name'] 106 | tool_args = tool_call['function']['arguments'] 107 | tool_args = json.loads(tool_args) 108 | if tool_name not in function_map: 109 | print(f"Error: {tool_name} is not a valid function name.") 110 | continue 111 | tool_func = function_map[tool_name] 112 | tool_response = tool_func(**tool_args) 113 | tools_response_messages.append({ 114 | "role": "tool", "content": json.dumps(tool_response) 115 | }) 116 | 117 | if tools_response_messages: 118 | request['messages'] += tools_response_messages 119 | response = requests.post( 120 | proxy_url, 121 | headers=header, 122 | json=request 123 | ) 124 | if response.status_code == 200: 125 | res = response.json() 126 | print(res['choices'][0]['message']['content']) 127 | else: 128 | print("Error:", response.status_code, response.text) 129 | else: 130 | print(message['content']) 131 | else: 132 | print("Error:", response.status_code, response.text) 133 | -------------------------------------------------------------------------------- /cookbook/function_call_without_schema.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | import os 4 | 5 | api_key=os.environ["GROQ_API_KEY"], 6 | header = { 7 | "Authorization": f"Bearer {api_key}", 8 | "Content-Type": "application/json" 9 | } 10 | 11 | proxy_url = "https://groqcall.ai/proxy/groq/v1/chat/completions" # or "http://localhost:8000/proxy/groq/v1/chat/completions" if running locally 12 | 13 | request = { 14 | "messages": [ 15 | { 16 | "role": "system", 17 | "content": "YOU MUST FOLLOW THESE INSTRUCTIONS CAREFULLY.\n\n1. Use markdown to format your answers.\n" 18 | }, 19 | { 20 | "role": "user", 21 | "content": "What's happening in France? Summarize top stories with sources, very short and concise." 22 | } 23 | ], 24 | "model": "mixtral-8x7b-32768", 25 | "tool_choice": "auto", 26 | "tools": [ 27 | { 28 | "type": "function", 29 | "function": { 30 | "name": "duckduck.search" 31 | } 32 | }, 33 | { 34 | "type": "function", 35 | "function": { 36 | "name": "duckduck.news" 37 | } 38 | } 39 | ] 40 | } 41 | 42 | response = requests.post( 43 | proxy_url, 44 | headers=header, 45 | json=request 46 | ) 47 | 48 | print(response.json()["choices"][0]["message"]["content"]) -------------------------------------------------------------------------------- /cookbook/functiona_call_groq_langchain.py: -------------------------------------------------------------------------------- 1 | # pip install --upgrade --quiet langchain-groq tavily-python langchain langchainhub langchain-openai 2 | 3 | from langchain_core.prompts import ChatPromptTemplate 4 | from langchain_groq import ChatGroq 5 | import os 6 | from dotenv import load_dotenv 7 | load_dotenv() 8 | from langchain import hub 9 | from langchain.agents import create_openai_tools_agent 10 | from langchain_community.tools.tavily_search import TavilySearchResults, TavilyAnswer 11 | from langchain.agents import AgentExecutor 12 | 13 | # The following code raise an error. 14 | chat = ChatGroq( 15 | temperature=0, 16 | groq_api_key=os.environ["GROQ_API_KEY"], 17 | model_name="mixtral-8x7b-32768", 18 | ) 19 | 20 | prompt = hub.pull("hwchase17/openai-tools-agent") 21 | tools = [TavilySearchResults(max_results=1)] 22 | agent = create_openai_tools_agent(chat, tools, prompt) 23 | 24 | agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=True, stream_runnable = False) 25 | agent_executor.invoke({"input": "What is Langchain?"}) 26 | 27 | 28 | # The following code works fine using GroqCall (Funckycall) proxy 29 | chat = ChatGroq( 30 | temperature=0, 31 | groq_api_key=os.environ["GROQ_API_KEY"], 32 | model_name="mixtral-8x7b-32768", 33 | groq_api_base= "http://localhost:8000/proxy/groqchain" 34 | # groq_api_base= "http://groqcall.ai/proxy/groqchain" 35 | ) 36 | 37 | # Example 1: Chat with tools 38 | prompt = hub.pull("hwchase17/openai-tools-agent") 39 | tools = [TavilySearchResults(max_results=1)] 40 | agent = create_openai_tools_agent(chat, tools, prompt) 41 | 42 | agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=True, stream_runnable = False) 43 | result = agent_executor.invoke({"input": "What is Langchain?"}) 44 | print(result) 45 | 46 | # Example 1: Simple chat 47 | system = "You are a helpful assistant." 48 | human = "{text}" 49 | prompt = ChatPromptTemplate.from_messages([("system", system), ("human", human)]) 50 | 51 | chain = prompt | chat 52 | result = chain.invoke({"text": "Explain the importance of low latency LLMs."}) 53 | print(result) 54 | -------------------------------------------------------------------------------- /cookbook/resources.py: -------------------------------------------------------------------------------- 1 | from phi.docker.app.postgres import PgVectorDb 2 | from phi.docker.resources import DockerResources 3 | 4 | # -*- PgVector2 running on port 5432:5432 5 | vector_db = PgVectorDb( 6 | name="knowledge-db", 7 | pg_user="ai", 8 | pg_password="ai", 9 | pg_database="ai", 10 | host_port=5532, 11 | ) 12 | 13 | # -*- DockerResources 14 | dev_docker_resources = DockerResources(apps=[vector_db]) 15 | -------------------------------------------------------------------------------- /examples/example_1.py: -------------------------------------------------------------------------------- 1 | from phi.llm.openai.like import OpenAILike 2 | from phi.assistant import Assistant 3 | from phi.tools.duckduckgo import DuckDuckGo 4 | import os, json 5 | 6 | groq = OpenAILike( 7 | model="mixtral-8x7b-32768", 8 | api_key=os.environ["GROQ_API_KEY"], 9 | base_url="https://api.groq.com/openai/v1" 10 | ) 11 | assistant = Assistant( 12 | llm=groq, 13 | tools=[DuckDuckGo()], show_tool_calls=True, markdown=True 14 | ) 15 | assistant.print_response("Whats happening in France? Summarize top stories with sources, very short and concise.", stream=False) 16 | 17 | 18 | 19 | 20 | my_groq = OpenAILike( 21 | # model="mixtral-8x7b-32768", 22 | model="gemma-7b-it", 23 | api_key=os.environ["GROQ_API_KEY"], 24 | base_url="https://groqcall.ai/proxy/groq/v1" 25 | ) 26 | assistant = Assistant( 27 | llm=my_groq, 28 | tools=[DuckDuckGo()], show_tool_calls=True, markdown=True 29 | ) 30 | assistant.print_response("Whats happening in France? Summarize top stories with sources, very short and concise.", stream=False) 31 | 32 | 33 | 34 | 35 | # Tried the proxy with Ollama and it works great, meaning we can use it with any provider. But, you never get the speed of Groq ;) 36 | # my_ollama = OpenAILike( 37 | # model="gemma:7b", 38 | # api_key="", 39 | # base_url="http://localhost:11235/proxy/ollama/v1" 40 | # ) 41 | # ollama_assistant = Assistant( 42 | # llm=my_ollama, 43 | # tools=[DuckDuckGo()], show_tool_calls=True, markdown=True 44 | # ) 45 | # ollama_assistant.print_response("Whats happening in France? Summarize top stories with sources, very short and concise.", stream=False) -------------------------------------------------------------------------------- /examples/example_2.py: -------------------------------------------------------------------------------- 1 | from duckduckgo_search import DDGS 2 | import requests, os 3 | api_key=os.environ["GROQ_API_KEY"] 4 | import json 5 | header = { 6 | "Authorization": f"Bearer {api_key}", 7 | "Content-Type": "application/json" 8 | } 9 | proxy_url = "https://groqcall.ai/proxy/groq/v1/chat/completions" 10 | 11 | 12 | def duckduckgo_search(query, max_results=None): 13 | """ 14 | Use this function to search DuckDuckGo for a query. 15 | """ 16 | with DDGS() as ddgs: 17 | return [r for r in ddgs.text(query, safesearch='off', max_results=max_results)] 18 | 19 | def duckduckgo_news(query, max_results=None): 20 | """ 21 | Use this function to get the latest news from DuckDuckGo. 22 | """ 23 | with DDGS() as ddgs: 24 | return [r for r in ddgs.news(query, safesearch='off', max_results=max_results)] 25 | 26 | function_map = { 27 | "duckduckgo_search": duckduckgo_search, 28 | "duckduckgo_news": duckduckgo_news, 29 | } 30 | 31 | request = { 32 | "messages": [ 33 | { 34 | "role": "system", 35 | "content": "YOU MUST FOLLOW THESE INSTRUCTIONS CAREFULLY.\n\n1. Use markdown to format your answers.\n" 36 | }, 37 | { 38 | "role": "user", 39 | "content": "Whats happening in France? Summarize top stories with sources, very short and concise." 40 | } 41 | ], 42 | "model": "mixtral-8x7b-32768", 43 | "tool_choice": "auto", 44 | "tools": [ 45 | { 46 | "type": "function", 47 | "function": { 48 | "name": "duckduckgo_search", 49 | "description": "Use this function to search DuckDuckGo for a query.\n\nArgs:\n query(str): The query to search for.\n max_results (optional, default=5): The maximum number of results to return.\n\nReturns:\n The result from DuckDuckGo.", 50 | "parameters": { 51 | "type": "object", 52 | "properties": { 53 | "query": { 54 | "type": "string" 55 | }, 56 | "max_results": { 57 | "type": [ 58 | "number", 59 | "null" 60 | ] 61 | } 62 | } 63 | } 64 | } 65 | }, 66 | { 67 | "type": "function", 68 | "function": { 69 | "name": "duckduckgo_news", 70 | "description": "Use this function to get the latest news from DuckDuckGo.\n\nArgs:\n query(str): The query to search for.\n max_results (optional, default=5): The maximum number of results to return.\n\nReturns:\n The latest news from DuckDuckGo.", 71 | "parameters": { 72 | "type": "object", 73 | "properties": { 74 | "query": { 75 | "type": "string" 76 | }, 77 | "max_results": { 78 | "type": [ 79 | "number", 80 | "null" 81 | ] 82 | } 83 | } 84 | } 85 | } 86 | } 87 | ] 88 | } 89 | 90 | response = requests.post( 91 | proxy_url, 92 | headers=header, 93 | json=request 94 | ) 95 | # Check if the request was successful 96 | if response.status_code == 200: 97 | # Process the response data (if needed) 98 | res = response.json() 99 | message = res['choices'][0]['message'] 100 | tools_response_messages = [] 101 | if not message['content'] and 'tool_calls' in message: 102 | for tool_call in message['tool_calls']: 103 | tool_name = tool_call['function']['name'] 104 | tool_args = tool_call['function']['arguments'] 105 | tool_args = json.loads(tool_args) 106 | if tool_name not in function_map: 107 | print(f"Error: {tool_name} is not a valid function name.") 108 | continue 109 | tool_func = function_map[tool_name] 110 | tool_response = tool_func(**tool_args) 111 | tools_response_messages.append({ 112 | "role": "tool", "content": json.dumps(tool_response) 113 | }) 114 | 115 | if tools_response_messages: 116 | request['messages'] += tools_response_messages 117 | response = requests.post( 118 | proxy_url, 119 | headers=header, 120 | json=request 121 | ) 122 | if response.status_code == 200: 123 | res = response.json() 124 | print(res['choices'][0]['message']['content']) 125 | else: 126 | print("Error:", response.status_code, response.text) 127 | else: 128 | print(message['content']) 129 | else: 130 | print("Error:", response.status_code, response.text) 131 | -------------------------------------------------------------------------------- /examples/example_3.py: -------------------------------------------------------------------------------- 1 | from duckduckgo_search import DDGS 2 | import requests, os 3 | api_key = os.environ["GROQ_API_KEY"] 4 | header = { 5 | "Authorization": f"Bearer {api_key}", 6 | "Content-Type": "application/json" 7 | } 8 | 9 | proxy_url = "https://groqcall.ai/proxy/groq/v1/chat/completions" 10 | 11 | 12 | request = { 13 | "messages": [ 14 | { 15 | "role": "system", 16 | "content": "YOU MUST FOLLOW THESE INSTRUCTIONS CAREFULLY.\n\n1. Use markdown to format your answers.\n", 17 | }, 18 | { 19 | "role": "user", 20 | "content": "Whats happening in France? Summarize top stories with sources, very short and concise. Also please search about the histoy of france as well.", 21 | }, 22 | ], 23 | "model": "mixtral-8x7b-32768", 24 | "tool_choice": "auto", 25 | "tools": [ 26 | { 27 | "type": "function", 28 | "function": { 29 | "name": "duckduck.search", 30 | }, 31 | }, 32 | { 33 | "type": "function", 34 | "function": { 35 | "name": "duckduck.news", 36 | }, 37 | }, 38 | ], 39 | } 40 | 41 | response = requests.post( 42 | proxy_url, 43 | headers=header, 44 | json=request, 45 | ) 46 | 47 | if response.status_code == 200: 48 | res = response.json() 49 | print(res["choices"][0]["message"]["content"]) 50 | else: 51 | print("Error:", response.status_code, response.text) 52 | -------------------------------------------------------------------------------- /examples/example_4.py: -------------------------------------------------------------------------------- 1 | from duckduckgo_search import DDGS 2 | import requests, os, json 3 | api_key = os.environ["GROQ_API_KEY"] 4 | header = { 5 | "Authorization": f"Bearer {api_key}", 6 | "Content-Type": "application/json" 7 | } 8 | proxy_url = "https://groqcall.ai/proxy/groq/v1/chat/completions" 9 | 10 | def duckduckgo_search(query, max_results=None): 11 | """ 12 | Use this function to search DuckDuckGo for a query. 13 | """ 14 | with DDGS() as ddgs: 15 | return [r for r in ddgs.text(query, safesearch='off', max_results=max_results)] 16 | 17 | function_map = { 18 | "duckduckgo_search": duckduckgo_search, 19 | } 20 | 21 | request = { 22 | "messages": [ 23 | { 24 | "role": "system", 25 | "content": "YOU MUST FOLLOW THESE INSTRUCTIONS CAREFULLY.\n\n1. Use markdown to format your answers.\n", 26 | }, 27 | { 28 | "role": "user", 29 | "content": "Whats happening in France? Summarize top stories with sources, very short and concise. Also please search about the histoy of france as well.", 30 | }, 31 | ], 32 | "model": "mixtral-8x7b-32768", 33 | "tool_choice": "auto", 34 | "tools": [ 35 | { 36 | "type": "function", 37 | "function": { 38 | "name": "duckduckgo_search", 39 | "description": "Use this function to search DuckDuckGo for a query.\n\nArgs:\n query(str): The query to search for.\n max_results (optional, default=5): The maximum number of results to return.\n\nReturns:\n The result from DuckDuckGo.", 40 | "parameters": { 41 | "type": "object", 42 | "properties": { 43 | "query": {"type": "string"}, 44 | "max_results": {"type": ["number", "null"]}, 45 | }, 46 | }, 47 | }, 48 | }, 49 | { 50 | "type": "function", 51 | "function": { 52 | "name": "duckduck.news", 53 | }, 54 | }, 55 | ], 56 | } 57 | 58 | response = requests.post( 59 | proxy_url, 60 | headers= header, 61 | json=request, 62 | ) 63 | # Check if the request was successful 64 | if response.status_code == 200: 65 | # Process the response data (if needed) 66 | res = response.json() 67 | message = res["choices"][0]["message"] 68 | tools_response_messages = [] 69 | if not message["content"] and "tool_calls" in message: 70 | if 'resolved' in res: 71 | # Append resolved message to the tools response messages 72 | tools_response_messages.extend(res['resolved']) 73 | 74 | for tool_call in message["tool_calls"]: 75 | tool_name = tool_call["function"]["name"] 76 | tool_args = tool_call["function"]["arguments"] 77 | tool_args = json.loads(tool_args) 78 | if tool_name not in function_map: 79 | print(f"Error: {tool_name} is not a valid function name.") 80 | continue 81 | tool_func = function_map[tool_name] 82 | tool_response = tool_func(**tool_args) 83 | tools_response_messages.append( 84 | {"role": "tool", "content": json.dumps(tool_response), "name": tool_name, "tool_call_id": tool_call["id"]} 85 | ) 86 | 87 | if tools_response_messages: 88 | request["messages"] += tools_response_messages 89 | response = requests.post( 90 | proxy_url, 91 | headers=header, 92 | json=request, 93 | ) 94 | if response.status_code == 200: 95 | res = response.json() 96 | print(res["choices"][0]["message"]["content"]) 97 | else: 98 | print("Error:", response.status_code, response.text) 99 | else: 100 | print(message["content"]) 101 | else: 102 | print("Error:", response.status_code, response.text) 103 | -------------------------------------------------------------------------------- /frontend/assets/README.md: -------------------------------------------------------------------------------- 1 | # GroqCall.ai - Lightning-Fast LLM Function Calls 2 | 3 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1q3is7qynCsx4s7FBznCfTMnokbKWIv1F?usp=sharing) 4 | [![Version](https://img.shields.io/badge/version-0.0.1-blue.svg)](https://github.com/unclecode/groqcall) 5 | [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 6 | 7 | GroqCall is a proxy server that enables lightning-fast function calls for Groq's Language Processing Unit (LPU) and other AI providers. It simplifies the creation of AI assistants by offering a wide range of built-in functions hosted on the cloud. 8 | 9 | ## Quickstart 10 | 11 | ### Using the Pre-built Server 12 | 13 | To quickly start using GroqCall without running it locally, make requests to one of the following base URLs: 14 | 15 | - Cloud: `https://groqcall.ai/proxy/groq/v1` 16 | - Local: `http://localhost:8000` (if running the proxy server locally) 17 | 18 | ### Running the Proxy Locally 19 | 20 | 1. Clone the repository: 21 | ``` 22 | git clone https://github.com/unclecode/groqcall.git 23 | cd groqcall 24 | ``` 25 | 26 | 2. Create and activate a virtual environment: 27 | ``` 28 | python -m venv venv 29 | source venv/bin/activate 30 | ``` 31 | 32 | 3. Install dependencies: 33 | ``` 34 | pip install -r requirements.txt 35 | ``` 36 | 37 | 4. Run the FastAPI server: 38 | ``` 39 | ./venv/bin/uvicorn --app-dir app/ main:app --reload 40 | ``` 41 | 42 | ## Examples 43 | 44 | ### Using GroqCall with PhiData 45 | 46 | ```python 47 | from phi.llm.openai.like import OpenAILike 48 | from phi.assistant import Assistant 49 | from phi.tools.duckduckgo import DuckDuckGo 50 | 51 | my_groq = OpenAILike( 52 | model="mixtral-8x7b-32768", 53 | api_key="YOUR_GROQ_API_KEY", 54 | base_url="https://groqcall.ai/proxy/groq/v1" # or "http://localhost:8000/proxy/groq/v1" if running locally 55 | ) 56 | 57 | assistant = Assistant( 58 | llm=my_groq, 59 | tools=[DuckDuckGo()], 60 | show_tool_calls=True, 61 | markdown=True 62 | ) 63 | 64 | assistant.print_response("What's happening in France? Summarize top stories with sources, very short and concise.", stream=False) 65 | ``` 66 | 67 | ### Using GroqCall with Requests 68 | 69 | #### FuncHub: Schema-less Function Calls 70 | 71 | GroqCall introduces FuncHub, which allows you to make function calls without passing the function schema. 72 | 73 | ```python 74 | import requests 75 | 76 | api_key = "YOUR_GROQ_API_KEY" 77 | header = { 78 | "Authorization": f"Bearer {api_key}", 79 | "Content-Type": "application/json" 80 | } 81 | 82 | proxy_url = "https://groqcall.ai/proxy/groq/v1/chat/completions" # or "http://localhost:8000/proxy/groq/v1/chat/completions" if running locally 83 | 84 | request = { 85 | "messages": [ 86 | { 87 | "role": "system", 88 | "content": "YOU MUST FOLLOW THESE INSTRUCTIONS CAREFULLY.\n\n1. Use markdown to format your answers.\n" 89 | }, 90 | { 91 | "role": "user", 92 | "content": "What's happening in France? Summarize top stories with sources, very short and concise." 93 | } 94 | ], 95 | "model": "mixtral-8x7b-32768", 96 | "tool_choice": "auto", 97 | "tools": [ 98 | { 99 | "type": "function", 100 | "function": { 101 | "name": "duckduck.search" 102 | } 103 | }, 104 | { 105 | "type": "function", 106 | "function": { 107 | "name": "duckduck.news" 108 | } 109 | } 110 | ] 111 | } 112 | 113 | response = requests.post( 114 | proxy_url, 115 | headers=header, 116 | json=request 117 | ) 118 | 119 | print(response.json()["choices"][0]["message"]["content"]) 120 | ``` 121 | 122 | - If you notice, the function schema is not passed in the request. This is because GroqCall uses FuncHub to automatically detect and call the function based on the function name in the cloud, Therefore you dont't need to parse the first response, call the function, and pass again. Check "functions" folder to add your own functions. I will create more examples in the close future to explain how to add your own functions. 123 | 124 | #### Passing Function Schemas 125 | 126 | If you prefer to pass your own function schemas, refer to the [Function Schema example](https://github.com/unclecode/groqcall/blob/main/cookbook/function_call_with_schema.py) in the cookbook. 127 | 128 | #### Rune proxy with Ollama locally 129 | 130 | Function call proxy can be used with Ollama. You should first install Ollama and run it locally. Then refer to the [Ollama example](https://github.com/unclecode/groqcall/blob/main/cookbook/function_call_ollama.py) in the cookbook. 131 | 132 | ## Cookbook 133 | 134 | Explore the [Cookbook](https://github.com/unclecode/groqcall/tree/main/cookbook) for more examples and use cases of GroqCall. 135 | 136 | ## Motivation 137 | 138 | Groq is a startup that designs highly specialized processor chips aimed specifically at running inference on large language models. They've introduced what they call the Language Processing Unit (LPU), and the speed is astounding—capable of producing 500 to 800 tokens per second or more. 139 | 140 | As an admirer of Groq and their community, I built this proxy to enable function calls using the OpenAI interface, allowing it to be called from any library. This engineering workaround has proven to be immensely useful in my company for various projects. 141 | 142 | ## Contributing 143 | 144 | Contributions are welcome! If you have ideas, suggestions, or would like to contribute to this project, please reach out to me on Twitter (X) @unclecode or via email at unclecode@kidocode.com. 145 | 146 | Let's collaborate and make this repository even more awesome! 🚀 147 | 148 | ## License 149 | 150 | This project is licensed under the Apache License 2.0. See [LICENSE](https://github.com/unclecode/groqcall/blob/main/LICENSE) for more information. -------------------------------------------------------------------------------- /frontend/assets/style.css: -------------------------------------------------------------------------------- 1 | @import "tailwindcss/base"; 2 | @import "tailwindcss/components"; 3 | @import "tailwindcss/utilities"; -------------------------------------------------------------------------------- /frontend/pages/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | GroqCall 10 | 11 | 12 | 31 | 32 | 33 |
34 |
35 |
36 |
37 |
38 |
Loading
39 |
40 |
41 |
42 |
43 | 44 | 100 | 101 | 102 | -------------------------------------------------------------------------------- /frontend/pages/index_old.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | GroqCall 10 | 11 | 15 | 16 | 17 | 18 | 46 | 47 | 48 |
49 |
50 |
51 |

GroqCall.ai

52 |

53 | 57 | Open In Colab 62 | 63 | 64 | Version 65 | 66 | 67 | License: MIT 72 | 73 |

74 |

75 | GroqCall is a proxy server that provides function calls for Groq's lightning-fast Language 76 | Processing Unit (LPU) and other AI providers. Additionally, the upcoming FuncyHub will offer a 77 | wide range of built-in functions, hosted on the cloud, making it easier to create AI assistants 78 | without the need to maintain function schemas in the codebase or execute them through multiple 79 | calls. 80 |

81 |

82 | Check github repo for more info: 83 | https://github.com/unclecode/groqcall 89 |

90 |

Motivation 🚀

91 |

92 | Groq is a startup that designs highly specialized processor chips aimed specifically at running 93 | inference on large language models. They've introduced what they call the Language Processing 94 | Unit (LPU), and the speed is astounding—capable of producing 500 to 800 tokens per second or 95 | more. I've become a big fan of Groq and their community; 96 |

97 |

98 | I admire what they're doing. It feels like after discovering electricity, the next challenge is 99 | moving it around quickly and efficiently. Groq is doing just that for Artificial Intelligence, 100 | making it easily accessible everywhere. They've opened up their API to the cloud, but as of now, 101 | they lack a function call capability. 102 |

103 |

104 | Unable to wait for this feature, I built a proxy that enables function calls using the OpenAI 105 | interface, allowing it to be called from any library. This engineering workaround has proven to 106 | be immensely useful in my company for various projects. Here's the link to the GitHub repository 107 | where you can explore and play around with it. I've included some examples in this collaboration 108 | for you to check out. 109 |

110 | Groq Chip 115 | Powered by Groq 120 | 121 |

Running the Proxy Locally 🖥️

122 |

To run this proxy locally on your own machine, follow these steps:

123 |
    124 |
  1. Clone the GitHub repository:
    2. 125 | 
    git clone https://github.com/unclecode/groqcall.git
    128 |
  2. Navigate to the project directory:
    3. 129 | 
    cd groqcall
    132 |
  3. Create a virtual environment:
    4. 133 | 
    python -m venv venv
    136 |
  4. Activate the virtual environment:
    5. 137 | 
    source venv/bin/activate
    140 |
  5. Install the required libraries:
    6. 141 | 
    pip install -r requirements.txt
    144 |
  6. Run the FastAPI server:
    7. 145 | 
    .venv/bin/uvicorn --app-dir app/ main:app --reload
    148 |
149 |

Using the Pre-built Server 🌐

150 |

151 | For your convenience, I have already set up a server that you can use temporarily. This allows 152 | you to quickly start using the proxy without having to run it locally. 153 |

154 |

155 | To use the pre-built server, simply make requests to the following base URL: 156 | https://groqcall.ai/proxy/groq/v1 157 |

158 | 159 |

Exploring GroqCall.ai 🚀

160 |

161 | This README is organized into three main sections, each showcasing different aspects of 162 | GroqCall.ai: 163 |

164 |
    165 | 166 |
  • 167 | Sending POST Requests: Here, I explore the functionality of sending direct 168 | POST requests to LLMs using GroqCall.ai. This section highlights the flexibility and 169 | control offered by the library when interacting with LLMs. 170 |
    • 171 |
  • 172 | FunckyHub: The third section introduces the concept of FunckyHub, a useful 173 | feature that simplifies the process of executing functions. With FunckyHub, there is no need 174 | to send the function JSON schema explicitly, as the functions are already hosted on the 175 | proxy server. This approach streamlines the workflow, allowing developers to obtain results 176 | with a single call without having to handle function calls in a production server. 177 |
    • 178 |
  • 179 | Using GroqCall with PhiData: In this section, I demonstrate how 180 | GroqCall.ai can be seamlessly integrated with other libraries such as my favorite one, the 181 | PhiData library, leveraging its built-in tools to connect to LLMs and perform external tool 182 | requests. 183 |
    • 184 |
185 | 
# The following libraries are optional if you're interested in using PhiData or managing your tools on the client side.
188 |     !pip install phidata > /dev/null
189 |     !pip install openai > /dev/null
190 |     !pip install duckduckgo-search > /dev/null
191 |
192 |

Sending POST request, with full functions implementation

193 |

194 | Check out the example file 195 | example_2.py 196 | for a full implementation of the following code. 197 |

198 | 
from duckduckgo_search import DDGS
199 |     import requests, os
200 |     import json
201 |     
202 |     # Here you pass your own GROQ API key
203 |     api_key=userdata.get("GROQ_API_KEY")
204 |     header = {
205 |         "Authorization": f"Bearer {api_key}",
206 |         "Content-Type": "application/json"
207 |     }
208 |     proxy_url = "https://groqcall.ai/proxy/groq/v1/chat/completions"
209 |     
210 |     
211 |     def duckduckgo_search(query, max_results=None):
212 |         """
213 |         Use this function to search DuckDuckGo for a query.
214 |         """
215 |         with DDGS() as ddgs:
216 |             return [r for r in ddgs.text(query, safesearch='off', max_results=max_results)]
217 |     
218 |     def duckduckgo_news(query, max_results=None):
219 |         """
220 |         Use this function to get the latest news from DuckDuckGo.
221 |         """
222 |         with DDGS() as ddgs:
223 |             return [r for r in ddgs.news(query, safesearch='off', max_results=max_results)]
224 |     
225 |     function_map = {
226 |         "duckduckgo_search": duckduckgo_search,
227 |         "duckduckgo_news": duckduckgo_news,
228 |     }
229 |     
230 |     request = {
231 |         "messages": [
232 |             {
233 |                 "role": "system",
234 |                 "content": "YOU MUST FOLLOW THESE INSTRUCTIONS CAREFULLY.\n<instructions>\n1. Use markdown to format your answers.\n</instructions>"
235 |             },
236 |             {
237 |                 "role": "user",
238 |                 "content": "Whats happening in France? Summarize top stories with sources, very short and concise."
239 |             }
240 |         ],
241 |         "model": "mixtral-8x7b-32768",
242 |         "tool_choice": "auto",
243 |         "tools": [
244 |             {
245 |                 "type": "function",
246 |                 "function": {
247 |                     "name": "duckduckgo_search",
248 |                     "description": "Use this function to search DuckDuckGo for a query.\n\nArgs:\n    query(str): The query to search for.\n    max_results (optional, default=5): The maximum number of results to return.\n\nReturns:\n    The result from DuckDuckGo.",
249 |                     "parameters": {
250 |                         "type": "object",
251 |                         "properties": {
252 |                             "query": {
253 |                                 "type": "string"
254 |                             },
255 |                             "max_results": {
256 |                                 "type": [
257 |                                     "number",
258 |                                     "null"
259 |                                 ]
260 |                             }
261 |                         }
262 |                     }
263 |                 }
264 |             },
265 |             {
266 |                 "type": "function",
267 |                 "function": {
268 |                     "name": "duckduckgo_news",
269 |                     "description": "Use this function to get the latest news from DuckDuckGo.\n\nArgs:\n    query(str): The query to search for.\n    max_results (optional, default=5): The maximum number of results to return.\n\nReturns:\n    The latest news from DuckDuckGo.",
270 |                     "parameters": {
271 |                         "type": "object",
272 |                         "properties": {
273 |                             "query": {
274 |                                 "type": "string"
275 |                             },
276 |                             "max_results": {
277 |                                 "type": [
278 |                                     "number",
279 |                                     "null"
280 |                                 ]
281 |                             }
282 |                         }
283 |                     }
284 |                 }
285 |             }
286 |         ]
287 |     }
288 |     
289 |     response = requests.post(
290 |         proxy_url,
291 |         headers=header,
292 |         json=request
293 |     )
294 |     if response.status_code == 200:
295 |         res = response.json()
296 |         message = res['choices'][0]['message']
297 |         tools_response_messages = []
298 |         if not message['content'] and 'tool_calls' in message:
299 |             for tool_call in message['tool_calls']:
300 |                 tool_name = tool_call['function']['name']
301 |                 tool_args = tool_call['function']['arguments']
302 |                 tool_args = json.loads(tool_args)
303 |                 if tool_name not in function_map:
304 |                     print(f"Error: {tool_name} is not a valid function name.")
305 |                     continue
306 |                 tool_func = function_map[tool_name]
307 |                 tool_response = tool_func(**tool_args)
308 |                 tools_response_messages.append({
309 |                     "role": "tool", "content": json.dumps(tool_response)
310 |                 })
311 |     
312 |             if tools_response_messages:
313 |                 request['messages'] += tools_response_messages
314 |                 response = requests.post(
315 |                     proxy_url,
316 |                     headers=header,
317 |                     json=request
318 |                 )
319 |                 if response.status_code == 200:
320 |                     res = response.json()
321 |                     print(res['choices'][0]['message']['content'])
322 |                 else:
323 |                     print("Error:", response.status_code, response.text)
324 |         else:
325 |             print(message['content'])
326 |     else:
327 |         print("Error:", response.status_code, response.text)
328 |
329 |

Schema-less Function Call 🤩

330 |

331 | Check out the example file 332 | example_3.py 333 | for a full implementation of the following code. 334 |

335 |

336 | In this method, we only need to provide the function's name, which consists of two parts, acting 337 | as a sort of namespace. The first part identifies the library or toolkit containing the 338 | functions, and the second part specifies the function's name, assuming it's already available on 339 | the proxy server. I aim to collaborate with the community to incorporate all typical functions, 340 | eliminating the need for passing a schema. Without having to handle function calls ourselves, a 341 | single request to the proxy enables it to identify and execute the functions, retrieve responses 342 | from large language models, and return the results to us. Thanks to Groq, all of this occurs in 343 | just seconds. 344 |

345 | 
from duckduckgo_search import DDGS
348 |     import requests, os
349 |     api_key = userdata.get("GROQ_API_KEY")
350 |     header = {
351 |         "Authorization": f"Bearer {api_key}",
352 |         "Content-Type": "application/json"
353 |     }
354 |     
355 |     proxy_url = "https://groqcall.ai/proxy/groq/v1/chat/completions"
356 |     
357 |     
358 |     request = {
359 |         "messages": [
360 |             {
361 |                 "role": "system",
362 |                 "content": "YOU MUST FOLLOW THESE INSTRUCTIONS CAREFULLY.\n<instructions>\n1. Use markdown to format your answers.\n</instructions>",
363 |             },
364 |             {
365 |                 "role": "user",
366 |                 "content": "Whats happening in France? Summarize top stories with sources, very short and concise. Also please search about the histoy of france as well.",
367 |             },
368 |         ],
369 |         "model": "mixtral-8x7b-32768",
370 |         "tool_choice": "auto",
371 |         "tools": [
372 |             {
373 |                 "type": "function",
374 |                 "function": {
375 |                     "name": "duckduck.search",
376 |                 },
377 |             },
378 |             {
379 |                 "type": "function",
380 |                 "function": {
381 |                     "name": "duckduck.news",
382 |                 },
383 |             },
384 |         ],
385 |     }
386 |     
387 |     response = requests.post(
388 |         proxy_url,
389 |         headers=header,
390 |         json=request,
391 |     )
392 |     
393 |     if response.status_code == 200:
394 |         res = response.json()
395 |         print(res["choices"][0]["message"]["content"])
396 |     else:
397 |         print("Error:", response.status_code, response.text)
398 |
399 |

Using with PhiData

400 |

401 | Check out the example file 402 | example_1.py 403 | for a full implementation of the following code. 404 |

405 |

406 | PhiData is a favorite of mine for creating AI assistants, thanks to its beautifully simplified 407 | interface, unlike the complexity seen in the LangChain library and LlamaIndex. I use it for many 408 | projects and want to give kudos to their team. It's open source, and I recommend everyone check 409 | it out. You can explore more from this link: 410 | https://github.com/phidatahq/phidata. 413 |

414 | 
from google.README import userdata
417 |     from phi.llm.openai.like import OpenAILike
418 |     from phi.assistant import Assistant
419 |     from phi.tools.duckduckgo import DuckDuckGo
420 |     import os, json
421 |     
422 |     
423 |     my_groq = OpenAILike(
424 |             model="mixtral-8x7b-32768",
425 |             api_key=userdata.get("GROQ_API_KEY"),
426 |             base_url="https://groqcall.ai/proxy/groq/v1"
427 |         )
428 |     assistant = Assistant(
429 |         llm=my_groq,
430 |         tools=[DuckDuckGo()], show_tool_calls=True, markdown=True
431 |     )
432 |     assistant.print_response("Whats happening in France? Summarize top stories with sources, very short and concise.", stream=False)
433 |     
434 |
435 |

Contributions Welcome! 🙌

436 |

437 | I am excited to extend and grow this repository by adding more built-in functions and 438 | integrating additional services. If you are interested in contributing to this project and being 439 | a part of its development, I would love to collaborate with you! I plan to create a discord 440 | channel for this project, where we can discuss ideas, share knowledge, and work together to 441 | enhance the repository. 442 |

443 |

Here's how you can get involved:

444 |
    445 |
  1. Fork the repository and create your own branch.
    2. 446 |
  2. 447 | Implement new functions, integrate additional services, or make improvements to the existing 448 | codebase. 449 |
    3. 450 |
  3. Test your changes to ensure they work as expected.
    4. 451 |
  4. Submit a pull request describing the changes you have made and why they are valuable.
    5. 452 |
453 |

454 | If you have any ideas, suggestions, or would like to discuss potential contributions, feel free 455 | to reach out to me. You can contact me through the following channels: 456 |

457 | 466 |

467 | I'm open to collaboration and excited to see how we can work together to enhance this project 468 | and provide value to the community. Let's connect and explore how we can help each other! 469 |

470 |

Together, let's make this repository even more awesome! 🚀

471 |
472 |
473 |
474 | 475 | 476 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aiohttp==3.9.3 2 | aiosignal==1.3.1 3 | annotated-types==0.6.0 4 | anyio==4.3.0 5 | appnope==0.1.4 6 | asttokens==2.4.1 7 | async-timeout==4.0.3 8 | attrs==23.2.0 9 | boto3==1.34.55 10 | botocore==1.34.55 11 | certifi==2024.2.2 12 | cffi==1.16.0 13 | charset-normalizer==3.3.2 14 | click==8.1.7 15 | comm==0.2.1 16 | curl_cffi==0.6.2 17 | debugpy==1.8.1 18 | decorator==5.1.1 19 | distro==1.9.0 20 | docker==7.0.0 21 | duckduckgo_search==4.5.0 22 | exceptiongroup==1.2.0 23 | executing==2.0.1 24 | fastapi==0.110.0 25 | filelock==3.13.1 26 | frozenlist==1.4.1 27 | fsspec==2024.2.0 28 | gitdb==4.0.11 29 | GitPython==3.1.42 30 | groq==0.4.2 31 | h11==0.14.0 32 | httpcore==1.0.4 33 | httpx==0.25.2 34 | huggingface-hub==0.21.3 35 | idna==3.6 36 | importlib-metadata==7.0.1 37 | ipykernel==6.29.3 38 | ipython==8.22.2 39 | jedi==0.19.1 40 | Jinja2==3.1.3 41 | jmespath==1.0.1 42 | jupyter_client==8.6.0 43 | jupyter_core==5.7.1 44 | litellm==1.31.8 45 | lxml==5.1.0 46 | markdown-it-py==3.0.0 47 | MarkupSafe==2.1.5 48 | matplotlib-inline==0.1.6 49 | mdurl==0.1.2 50 | mistralai==0.1.3 51 | multidict==6.0.5 52 | nest-asyncio==1.6.0 53 | numpy==1.26.4 54 | ollama==0.1.7 55 | openai==1.13.3 56 | orjson==3.9.15 57 | packaging==23.2 58 | pandas==2.2.1 59 | parso==0.8.3 60 | pexpect==4.9.0 61 | phidata==2.3.50 62 | platformdirs==4.2.0 63 | prompt-toolkit==3.0.43 64 | psutil==5.9.8 65 | ptyprocess==0.7.0 66 | pure-eval==0.2.2 67 | pyarrow==15.0.0 68 | pycparser==2.21 69 | pydantic==2.6.3 70 | pydantic-settings==2.2.1 71 | pydantic_core==2.16.3 72 | Pygments==2.17.2 73 | python-dateutil==2.9.0.post0 74 | python-dotenv==1.0.1 75 | pytz==2024.1 76 | PyYAML==6.0.1 77 | pyzmq==25.1.2 78 | regex==2023.12.25 79 | replicate==0.24.0 80 | requests==2.31.0 81 | rich==13.7.1 82 | s3transfer==0.10.0 83 | six==1.16.0 84 | smmap==5.0.1 85 | sniffio==1.3.1 86 | stack-data==0.6.3 87 | starlette==0.36.3 88 | tiktoken==0.6.0 89 | tokenizers==0.15.2 90 | tomli==2.0.1 91 | tornado==6.4 92 | tqdm==4.66.2 93 | traitlets==5.14.1 94 | typer==0.9.0 95 | typing_extensions==4.10.0 96 | tzdata==2024.1 97 | urllib3==2.0.7 98 | uvicorn==0.27.1 99 | wcwidth==0.2.13 100 | yarl==1.9.4 101 | zipp==3.17.0 102 | --------------------------------------------------------------------------------