├── .gitignore ├── LICENSE ├── README.md ├── assets ├── fonts │ ├── CabinSketch-Bold.ttf │ ├── CabinSketch-Regular.ttf │ └── HomemadeApple-Regular.ttf ├── icon.png ├── logos │ ├── combined_logos.png │ ├── mlx.png │ ├── mlx_logo.png │ ├── ollama.png │ ├── pyollama_1 copy.jpeg │ ├── pyollama_1.png │ ├── pyollama_2.jpeg │ ├── readme_logo.png │ └── vk_logo.png ├── pyollamx_sample.png └── pyollamx_sample_updated.png ├── buildflet.sh ├── create-dmg.sh ├── ddg_test.py ├── history.py ├── icon.icns ├── llava_test.py ├── main.py ├── mlxClient.py ├── mlxLLM.py ├── mlxLLM_local.py ├── model_hub.py ├── models.py ├── ollama ├── __init__.py ├── _client.py └── _types.py ├── ollamaClient.py ├── ollamaOpenAIClient.py ├── prompt.py ├── prompts.txt ├── pyomlx_test.py ├── reqs.txt ├── requirements copy.txt ├── requirements.txt ├── search.py ├── settings.py ├── test.py ├── test123.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | 162 | **/.DS_Store -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Viz 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![](assets/logos/combined_logos.png) 2 | # PyOllaMx (Ollama + MlX) 3 | #### `Your gateway to both Ollama & Apple MlX models` 4 | 5 | ### ![Downloads](https://img.shields.io/github/downloads/kspviswa/pyOllaMx/total.svg) 6 | 7 | Inspired by [Ollama](https://github.com/ollama/ollama), [Apple MlX](https://github.com/ml-explore/mlx) projects and frustrated by the dependencies from external applications like Bing, Chat-GPT etc, I wanted to have my own personal chatbot as a native MacOS application. Sure there are alternatives like streamlit, gradio (which are based, thereby needing a browser) or others like Ollamac, LMStudio, mindmac etc which are good but then restrictive in some means (either by license, or paid or not versatile). Also I wanted to enjoy both Ollama (based on `llama.cpp`) and Mlx models (which are suitable for image generation, audio generation etc and heck I own a mac with Apple silicon 👨🏻‍💻) through a single uniform interface. 8 | 9 | All these lead to this project (PyOllaMx) and another sister project called [PyOMlx](https://github.com/kspviswa/PyOMlx). 10 | 11 | I'm using these in my day to day workflow and I intend to keep develop these for my use and benefit. 12 | 13 | If you find this valuable, feel free to use it and contribute to this project as well. Please ⭐️ this repo to show your support and make my day! 14 | 15 | I'm planning on work on next items on this [roadmap.md](roadmap.md). Feel free to comment your thoughts (if any) and influence my work (if interested) 16 | 17 | MacOS DMGs are available in [Releases](https://github.com/kspviswa/pyOllaMx/releases) 18 | 19 | ## PyOllaMx vs PyOMlx 20 | 21 | [PyOllaMx](https://github.com/kspviswa/pyOllaMx) : ChatBot application capable of chatting with both Ollama and Apple MlX models. For this app to function, it needs both [Ollama](https://github.com/ollama/ollama) & [PyOMlx](https://github.com/kspviswa/PyOMlx) macos app running. These 2 apps will serve their respective models on localhost for PyOllaMx to chat. 22 | 23 | [PyOMlx](https://github.com/kspviswa/PyOMlx) : A Macos App capable of discovering, loading & serving Apple MlX models downloaded from [Apple MLX Community repo in hugging face](https://huggingface.co/mlx-community) 🤗 24 | 25 | ## Star History 26 | 27 | [![Star History Chart](https://api.star-history.com/svg?repos=kspviswa/pyOllaMx&type=Date)](https://star-history.com/#kspviswa/pyOllaMx&Date) 28 | 29 | ## How to use? 30 | 31 | 1) Install [Ollama Application](https://ollama.ai/download) & use Ollama CLI to download your desired models 32 | ``` 33 | ollama pull 34 | ollama pull mistral 35 | ``` 36 | This command will download the Ollama models in a known location to PyOllaMx 37 | 38 | > [!TIP] 39 | > As of PyOllaMx v0.0.4, you can download & manage ollama models right within PyOllaMx's ModelHub. Check the [v0.0.4 release page](https://github.com/kspviswa/pyOllaMx/releases/tag/v0.0.4) for more details 40 | 41 | 2) Install [MlX Models from Hugging Face repo](https://huggingface.co/mlx-community). 42 | 43 | use hugging-face cli 44 | ``` 45 | pip install huggingface_hub hf_transfer 46 | 47 | export HF_HUB_ENABLE_HF_TRANSFER=1 48 | huggingface-cli download mlx-community/CodeLlama-7b-Python-4bit-MLX 49 | ``` 50 | This command will download the MlX models in a known location to PyOllaMx 51 | 52 | 3) Now simply open the **PyOllaMx** and start chatting 53 | 54 | ![sample](assets/pyollamx_sample_updated.png) 55 | 56 | ## [v0.0.7 Features](https://github.com/kspviswa/pyOllaMx/releases/tag/v0.0.7) 57 | 58 | ### New Functionality 59 | 60 | Added supported to **thinking tokens** for reasoning models like DeepSeek-R1 61 | 62 | ## [v0.0.4 Features](https://github.com/kspviswa/pyOllaMx/releases/tag/v0.0.4) 63 | 64 | ### New Functionality 65 | Now you can download Ollama models right within 🤌🏻 PyOllaMx's Model Hub tab. You can also inspect existing models 🧐, delete models 🗑️ right within PyOllaMx instead of using Ollama CLI. This greatly simplifies the user experience 🤩🤩. And you before you ask, yes I'm working to bring similar functionality for MLX models from huggingface hub. Please stay tuned 😎 66 | 67 | ### BugFixes 68 | 1. Updated DDGS dependency to fix some of the rate limit issues 69 | 70 | Click the release version link above ☝🏻 to view demo gifs explaining the features. 71 | 72 | ## [v0.0.3 Features](https://github.com/kspviswa/pyOllaMx/releases/tag/v0.0.3) 73 | 74 | 1. Dark mode support - Toggle between Dark & Light mode with a click of the icon 75 | 2. Model settings menu - Brand new settings menu to set the model name and the temperature along with Ollama & MlX model toggle 76 | 3. Streaming support - Streaming support for both chat & search tasks 77 | 4. Brand New Status bar - Status bar that displays the selected mode name, model type & model temperature 78 | 5. Web search enabled for Apple MlX models - Now you can use Apple MlX models to power the web search when choosing the search tab 79 | 80 | Click the release version link above ☝🏻 to view demo gifs explaining the features. 81 | 82 | ## [v0.0.2 Features](https://github.com/kspviswa/pyOllaMx/releases/tag/v0.0.2) 83 | 84 | 1. Web search capability _(powered by DuckDuckGo search engine via https://github.com/deedy5/duckduckgo_search)_ 85 | a. Web search powered via basic RAG using prompt engineering. More advanced techniques are in pipeline 86 | b. Search response will cite clickable sources for easy follow-up / deep dive 87 | c. Beneath every search response, search keywords are also shown to verify the search scope 88 | d. Easy toggle between chat and search operations 89 | 2. Clear / Erase history 90 | 3. Automatic scroll on chat messages for better user experience 91 | 4. Basic error & exception handling for searches 92 | 93 | Limitations: 94 | 95 | - Web search only enabled for Ollama models. Use dolphin-mistral:7b model for better results. MlX model support is planned for next release 96 | - Search results aren't deterministic and vary vastly among the chosen models. So play with different models to find your optimum 97 | - Sometimes search results are gibberish. It is due to the fact that search engine RAG is vanilla i.e done via basic prompt engineering without any library support. So re-trigger the same search prompt and see the response once again if the results aren't satisfactory. 98 | 99 | Click the release version link above ☝🏻 to view demo gifs explaining the features. 100 | 101 | ## [v0.0.1 Features](https://github.com/kspviswa/pyOllaMx/releases/tag/v0.0.1) 102 | 103 | - Auto discover Ollama & MlX models. Simply download the models as you do with respective tools and pyOllaMx would pull the models seamlessly 104 | - Markdown support on chat messages for programming code 105 | - Selectable Text 106 | - Temperature control 107 | -------------------------------------------------------------------------------- /assets/fonts/CabinSketch-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kspviswa/pyOllaMx/d12f5268fec92928ea0b550f45e171ac773cd836/assets/fonts/CabinSketch-Bold.ttf -------------------------------------------------------------------------------- /assets/fonts/CabinSketch-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kspviswa/pyOllaMx/d12f5268fec92928ea0b550f45e171ac773cd836/assets/fonts/CabinSketch-Regular.ttf -------------------------------------------------------------------------------- /assets/fonts/HomemadeApple-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kspviswa/pyOllaMx/d12f5268fec92928ea0b550f45e171ac773cd836/assets/fonts/HomemadeApple-Regular.ttf -------------------------------------------------------------------------------- /assets/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kspviswa/pyOllaMx/d12f5268fec92928ea0b550f45e171ac773cd836/assets/icon.png -------------------------------------------------------------------------------- /assets/logos/combined_logos.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kspviswa/pyOllaMx/d12f5268fec92928ea0b550f45e171ac773cd836/assets/logos/combined_logos.png -------------------------------------------------------------------------------- /assets/logos/mlx.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kspviswa/pyOllaMx/d12f5268fec92928ea0b550f45e171ac773cd836/assets/logos/mlx.png -------------------------------------------------------------------------------- /assets/logos/mlx_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kspviswa/pyOllaMx/d12f5268fec92928ea0b550f45e171ac773cd836/assets/logos/mlx_logo.png -------------------------------------------------------------------------------- /assets/logos/ollama.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kspviswa/pyOllaMx/d12f5268fec92928ea0b550f45e171ac773cd836/assets/logos/ollama.png -------------------------------------------------------------------------------- /assets/logos/pyollama_1 copy.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kspviswa/pyOllaMx/d12f5268fec92928ea0b550f45e171ac773cd836/assets/logos/pyollama_1 copy.jpeg -------------------------------------------------------------------------------- /assets/logos/pyollama_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kspviswa/pyOllaMx/d12f5268fec92928ea0b550f45e171ac773cd836/assets/logos/pyollama_1.png -------------------------------------------------------------------------------- /assets/logos/pyollama_2.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kspviswa/pyOllaMx/d12f5268fec92928ea0b550f45e171ac773cd836/assets/logos/pyollama_2.jpeg -------------------------------------------------------------------------------- /assets/logos/readme_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kspviswa/pyOllaMx/d12f5268fec92928ea0b550f45e171ac773cd836/assets/logos/readme_logo.png -------------------------------------------------------------------------------- /assets/logos/vk_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kspviswa/pyOllaMx/d12f5268fec92928ea0b550f45e171ac773cd836/assets/logos/vk_logo.png -------------------------------------------------------------------------------- /assets/pyollamx_sample.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kspviswa/pyOllaMx/d12f5268fec92928ea0b550f45e171ac773cd836/assets/pyollamx_sample.png -------------------------------------------------------------------------------- /assets/pyollamx_sample_updated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kspviswa/pyOllaMx/d12f5268fec92928ea0b550f45e171ac773cd836/assets/pyollamx_sample_updated.png -------------------------------------------------------------------------------- /buildflet.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | flet build macos --build-version '0.0.7' --copyright 'Viswa Kumar 2025 ©' --project 'PyOllaMx' --company 'PyOllaMx' -------------------------------------------------------------------------------- /create-dmg.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Create a folder (named dmg) to prepare our DMG in (if it doesn't already exist). 3 | mkdir -p dist/dmg 4 | # Empty the dmg folder. 5 | rm -r dist/dmg/* 6 | # Copy the app bundle to the dmg folder. 7 | cp -r "build/macos/PyOllamx.app" dist/dmg 8 | # If the DMG already exists, delete it. 9 | test -f "dist/PyOllaMx.dmg" && rm "dist/PyOllaMx.dmg" 10 | create-dmg \ 11 | --volname "PyOllaMx" \ 12 | --volicon "icon.icns" \ 13 | --window-pos 200 120 \ 14 | --window-size 600 300 \ 15 | --icon-size 100 \ 16 | --icon "PyOllaMx.app" 175 120 \ 17 | --hide-extension "PyOllaMx.app" \ 18 | --app-drop-link 425 120 \ 19 | "dist/PyOllaMx.dmg" \ 20 | "dist/dmg/" -------------------------------------------------------------------------------- /ddg_test.py: -------------------------------------------------------------------------------- 1 | import threading 2 | from duckduckgo_search import DDGS 3 | 4 | def do_Search(): 5 | with DDGS() as ddgs: 6 | searchResults = [r for r in ddgs.text('What is duckduckgo?', max_results=5)] 7 | print(searchResults) 8 | 9 | def main(): 10 | x = threading.Thread(target=do_Search) 11 | x.start() 12 | x.join() 13 | #do_Search() 14 | 15 | if __name__ == '__main__': 16 | main() -------------------------------------------------------------------------------- /history.py: -------------------------------------------------------------------------------- 1 | import flet as ft 2 | 3 | history_banner_text = ft.Text(value='Conversation History', style=ft.TextStyle(font_family='CabinSketch-Bold'), size=30) 4 | history_banner_image = ft.Image(src=f"logos/combined_logos.png", 5 | width=75, 6 | height=75, 7 | fit=ft.ImageFit.CONTAIN, 8 | ) 9 | 10 | history_banner_view = ft.Row([ 11 | history_banner_image, 12 | history_banner_text 13 | ], alignment=ft.MainAxisAlignment.CENTER, vertical_alignment=ft.CrossAxisAlignment.CENTER) 14 | 15 | history_mock_data = ft.DataTable( 16 | columns=[ 17 | ft.DataColumn(ft.Text("Date/Time")), 18 | ft.DataColumn(ft.Text("Topic")), 19 | ft.DataColumn(ft.Text("Content"), numeric=True), 20 | ], 21 | rows=[ 22 | ft.DataRow( 23 | cells=[ 24 | ft.DataCell(ft.Text("April 12")), 25 | ft.DataCell(ft.Text("Topic 1")), 26 | ft.DataCell(ft.Text(f""" 27 | User : xxxxx 28 | AI : xxxxx 29 | User : xxxxx 30 | AI : xxxxx 31 | User : xxxxx 32 | AI : xxxxx 33 | """)), 34 | ], 35 | ), 36 | ft.DataRow( 37 | cells=[ 38 | ft.DataCell(ft.Text("April 12")), 39 | ft.DataCell(ft.Text("Topic 1")), 40 | ft.DataCell(ft.Text(f""" 41 | User : xxxxx 42 | AI : xxxxx 43 | User : xxxxx 44 | AI : xxxxx 45 | User : xxxxx 46 | AI : xxxxx 47 | """)), 48 | ], 49 | ), 50 | ft.DataRow( 51 | cells=[ 52 | ft.DataCell(ft.Text("April 12")), 53 | ft.DataCell(ft.Text("Topic 1")), 54 | ft.DataCell(ft.Text(f""" 55 | User : xxxxx 56 | AI : xxxxx 57 | User : xxxxx 58 | AI : xxxxx 59 | User : xxxxx 60 | AI : xxxxx 61 | """)), 62 | ], 63 | ), 64 | 65 | ], 66 | ) 67 | 68 | history_data_view = ft.Container(content=history_mock_data) 69 | 70 | coming_soon_view = ft.Column([ 71 | ft.Text(value='Coming Soon!', style=ft.TextStyle(font_family='CabinSketch-Bold'), size=50) 72 | ], alignment=ft.MainAxisAlignment.CENTER, horizontal_alignment=ft.CrossAxisAlignment.CENTER) 73 | 74 | 75 | def historyView(theme: str) -> ft.View: 76 | 77 | return ft.View( 78 | "/history", 79 | controls = [ 80 | ft.AppBar(title=""), 81 | history_banner_view, 82 | coming_soon_view, 83 | #history_data_view 84 | ] 85 | ) -------------------------------------------------------------------------------- /icon.icns: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kspviswa/pyOllaMx/d12f5268fec92928ea0b550f45e171ac773cd836/icon.icns -------------------------------------------------------------------------------- /llava_test.py: -------------------------------------------------------------------------------- 1 | import ollama 2 | 3 | res = ollama.chat( 4 | model="llava:7b", 5 | messages=[ 6 | { 7 | 'role': 'user', 8 | 'content': 'Describe this image:', 9 | 'images': ['assets/icon.png'] 10 | } 11 | ] 12 | ) 13 | 14 | print(res['message']['content']) -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import flet as ft 2 | from flet_core.control_event import ControlEvent 3 | from prompt import * 4 | from models import * 5 | from utils import Avatar, Message 6 | from model_hub import * 7 | from settings import * 8 | from history import * 9 | import time 10 | 11 | 12 | def main(page: ft.Page) -> None: 13 | page.title = 'PyOllaMx' 14 | page.theme_mode = 'light' 15 | page.scroll = ft.ScrollMode.ADAPTIVE 16 | #page.bgcolor = '#C7F9D6' 17 | page.window.resizable = False 18 | 19 | page.theme = ft.Theme( 20 | font_family="CabinSketch-Regular", 21 | color_scheme=ft.ColorScheme(primary='black') 22 | ) 23 | page.dark_theme = ft.Theme( 24 | font_family="CabinSketch-Regular", 25 | color_scheme=ft.ColorScheme(primary='#ffde03') 26 | ) 27 | 28 | page.window.height = 880 29 | page.window.width= 872 30 | 31 | #page.theme = ft.theme.Theme(font_family="CabinSketch-Regular") 32 | page.fonts = { 33 | "Roboto Mono": "RobotoMono-VariableFont_wght.ttf", 34 | "Homemade Apple" : "fonts/HomemadeApple-Regular.ttf", 35 | "CabinSketch-Bold" : "fonts/CabinSketch-Bold.ttf", 36 | "CabinSketch-Regular" : "fonts/CabinSketch-Regular.ttf" 37 | } 38 | 39 | #initialize page state 40 | page.session.set('selected_model', 'N/A') 41 | page.session.set('selected_temp', 0.0) 42 | page.session.set('isMlx', False) 43 | 44 | banner_image = ft.Image(src=f"logos/pyollama_1.png", 45 | width=75, 46 | height=75, 47 | fit=ft.ImageFit.CONTAIN, 48 | ) 49 | banner_text = ft.Text(value='PyOllaMx', style=ft.TextStyle(font_family='CabinSketch-Bold'), size=30) 50 | subbanner_text = ft.Text(value='Your gateway to both Ollama & Apple MlX models') 51 | chat_messages = ft.Column( 52 | alignment=ft.MainAxisAlignment.CENTER, 53 | horizontal_alignment=ft.CrossAxisAlignment.START, 54 | scroll=ft.ScrollMode.ADAPTIVE, 55 | height=450, 56 | width=695, 57 | ) 58 | search_messages = ft.Column( 59 | alignment=ft.MainAxisAlignment.CENTER, 60 | horizontal_alignment=ft.CrossAxisAlignment.START, 61 | scroll=ft.ScrollMode.ADAPTIVE, 62 | height=450, 63 | width=695, 64 | ) 65 | tabs = ft.Tabs( 66 | selected_index=0, 67 | animation_duration=300, 68 | tabs=[ 69 | ft.Tab( 70 | text="Chat", 71 | icon=ft.icons.CHAT, 72 | content = chat_messages, 73 | ), 74 | ft.Tab( 75 | text="Search", 76 | icon=ft.icons.TRAVEL_EXPLORE_OUTLINED, 77 | content = search_messages 78 | ), 79 | ], 80 | height=488, 81 | width=700, 82 | scrollable=True, 83 | ) 84 | 85 | def get_check_color() -> str: 86 | if page.theme_mode == 'dark' : 87 | print('dark') 88 | return 'black' 89 | else: 90 | print('light') 91 | return 'white' 92 | 93 | user_text = ft.Text(value='Enter your prompt', style=ft.TextStyle(font_family='CabinSketch-Bold')) 94 | enable_streaming = ft.CupertinoCheckbox(label='Enable Streaming', value=False, check_color=get_check_color()) 95 | user_text_field = ft.TextField(multiline=True, 96 | width=675, autofocus=True, label='Enter your prompt') 97 | user_text_field.border_color = 'white' if page.theme_mode == 'dark' else 'black' 98 | send_button = ft.ElevatedButton("Send", icon=ft.icons.ROCKET_LAUNCH, tooltip='Select a model in the settings menu & type in a prompt to enable this control') 99 | clear_button = ft.ElevatedButton("chats", icon=ft.icons.DELETE_FOREVER, icon_color="pink600", tooltip='Atleast one response from AI should be available to delete this conversation') 100 | pr = ft.ProgressRing(width=16, height=16, stroke_width=2) 101 | pr.value = 0 102 | pr_ph = ft.Text() 103 | model_dropdown = ft.Dropdown( 104 | label = "Load a model^", 105 | hint_text= "Choose from available models", 106 | options = retModelOptions(), 107 | value = "unselected", 108 | dense=True, 109 | focused_bgcolor='pink', 110 | ) 111 | select_mlX_models = ft.Switch(label='Load 🖥️ mlX models from HF', 112 | value=False, 113 | adaptive=True, 114 | label_position=ft.LabelPosition.LEFT) 115 | temp_label = ft.Text(value='Temperature') 116 | temp_slider = ft.CupertinoSlider(min=0.0, max=1.0, divisions=10, value=0.3) 117 | temp_value_label = ft.Text(value=temp_slider.value) 118 | model_help_text = ft.Text('^Ollama models are loaded by default', style=ft.TextStyle(size=10), text_align=ft.TextAlign.LEFT) 119 | 120 | # BottomBar 121 | selected_model = ft.Text('None', style=ft.TextStyle(size=15)) 122 | selected_temp = ft.Text('N/A', style=ft.TextStyle(size=15)) 123 | selected_model_image = ft.Image(src='logos/combined_logos.png', width=35, height=35) 124 | bottom_control = ft.Row([ 125 | ft.Text('Model Selected: ', style=ft.TextStyle(size=15)), 126 | selected_model, 127 | selected_model_image, 128 | ft.Text('Temperature: ', style=ft.TextStyle(size=15)), 129 | selected_temp 130 | ], alignment=ft.MainAxisAlignment.START, vertical_alignment=ft.CrossAxisAlignment.CENTER) 131 | bottom_appbar = ft.BottomAppBar(content=bottom_control) 132 | 133 | def open_url(e): 134 | page.launch_url(e.data) 135 | 136 | def updateChat(message: Message, controlHandle: ft.Column, ai_response: bool = False): 137 | #print(f'message {message} ai_response {ai_response} controlHandle {controlHandle}') 138 | if ai_response: 139 | ai_thinking_text = re.sub(r".*$", "", message.text, flags=re.DOTALL) if "" in message.text else "" 140 | ai_non_thinking_text = message.text 141 | 142 | #print(f'### {ai_thinking_text}') 143 | ai_message_container = ft.Container(width=550) 144 | ai_message_md = ft.Markdown(value="", extension_set="gitHubWeb", code_theme='obsidian', code_style=ft.TextStyle(font_family='Roboto Mono'), on_tap_link=open_url, auto_follow_links=True) 145 | ai_message_md_selectable = ft.SelectionArea(content=ai_message_md) 146 | ai_message_thinking_md = ft.ExpansionTile( 147 | title=ft.Text("Thinking tokens 🤔", font_family="RobotoSlab"), 148 | subtitle=ft.Text("Expand to reveal the model's thinking tokens", theme_style=ft.TextThemeStyle.BODY_SMALL, font_family="RobotoSlab"), 149 | affinity=ft.TileAffinity.LEADING, 150 | initially_expanded=False, 151 | collapsed_text_color=ft.Colors.BLUE, 152 | text_color=ft.Colors.BLUE, 153 | controls=[ 154 | ft.ListTile(title=ft.Text( 155 | theme_style=ft.TextThemeStyle.BODY_SMALL, 156 | font_family="RobotoSlab", 157 | )), 158 | ], 159 | ) 160 | if ai_thinking_text : 161 | ai_message_container.content = ft.Column([ai_message_thinking_md, 162 | ai_message_md_selectable, 163 | ]) 164 | ai_non_thinking_text = ''.join(message.text.split("")[1:]) 165 | else: 166 | ai_message_container.content = ai_message_md_selectable 167 | 168 | controlHandle.controls.append( 169 | ft.Row([ 170 | ft.Image(src=getAILogo(page.session.get('isMlx')), 171 | width=50, 172 | height=50, 173 | fit=ft.ImageFit.CONTAIN), 174 | ai_message_container 175 | ], 176 | width=500, vertical_alignment=ft.CrossAxisAlignment.START, 177 | ) 178 | ) 179 | ai_message_thinking_md.controls[0].title.value = ai_thinking_text 180 | if enable_streaming.value: 181 | full_r = "" 182 | for chunk in ai_non_thinking_text.split(sep=" "): 183 | full_r += chunk + " " 184 | ai_message_md.value = full_r 185 | # controlHandle.scroll_to(offset=-1, duration=100, curve=ft.AnimationCurve.EASE_IN_OUT) 186 | page.update() 187 | time.sleep(0.05) 188 | else: 189 | ai_message_md.value = ai_non_thinking_text 190 | else: 191 | controlHandle.controls.append( 192 | ft.Row([ 193 | ft.Image(src=f"logos/vk_logo.png", 194 | width=50, 195 | height=50, 196 | fit=ft.ImageFit.CONTAIN), 197 | ft.Container(content=ft.Text(message.text, selectable=True), width=500) 198 | ], 199 | auto_scroll=True, 200 | width=500 201 | ) 202 | ) 203 | #controlHandle.scroll_to(offset=-1, duration=100, curve=ft.AnimationCurve.EASE_IN_OUT) 204 | page.update() 205 | 206 | def getAILogo(isMlx: bool) -> str: 207 | return f'logos/mlx_logo.png' if isMlx else f'logos/pyollama_1.png' 208 | 209 | def getBottomBarModelLogo(isMlx: bool) -> str: 210 | return f'logos/mlx.png' if isMlx else f'logos/ollama.png' 211 | 212 | def show_spinning(): 213 | pr_ph.value='Working...🏃🏻‍♂️⏳' 214 | pr.value = None 215 | user_text_field.disabled = True 216 | send_button.disabled = True 217 | clear_button.disabled = True 218 | page.update() 219 | 220 | def end_spinning(): 221 | pr_ph.value="" 222 | pr.value = 0 223 | user_text_field.disabled = False 224 | send_button.disabled = False 225 | clear_button.disabled = False 226 | page.update() 227 | 228 | def send(e: ControlEvent) -> None: 229 | prompt = user_text_field.value 230 | isChat = True if tabs.selected_index == 0 else False 231 | ctrlHandle = chat_messages if isChat else search_messages 232 | updateChat(Message(user='user', text=prompt), controlHandle=ctrlHandle, ai_response=False) 233 | user_text_field.value = "" 234 | show_spinning() 235 | isMlx = page.session.get('isMlx') 236 | res, keys = firePrompt(prompt=prompt, model=page.session.get('selected_model'), temp=page.session.get('selected_temp'), isMlx=isMlx, chat_mode=isChat) 237 | if keys != "": 238 | res = res + f'\n\n Keywords used for search : {keys} ' 239 | updateChat(Message(user='assistant', text=res), controlHandle=ctrlHandle, ai_response=True) 240 | end_spinning() 241 | page.update() 242 | 243 | def enableSend(e: ControlEvent) -> None: 244 | if user_text_field.value != "" and page.session.get('selected_model') != "N/A": 245 | send_button.disabled = False 246 | clear_button.disabled = False 247 | page.update() 248 | 249 | def swapModels(e: ControlEvent) -> None: 250 | if select_mlX_models.value: 251 | model_dropdown.options = retModelOptions(True) 252 | else: 253 | model_dropdown.options = retModelOptions() 254 | banner_image.src = getAILogo(select_mlX_models.value) 255 | page.update() 256 | 257 | def displayTemp(e: ControlEvent) -> None: 258 | temp_value_label.value = temp_slider.value 259 | page.update() 260 | 261 | def clear(e: ControlEvent) -> None: 262 | if tabs.selected_index == 0: 263 | del chat_messages.controls[:] 264 | clearChatHistory() 265 | else: 266 | del search_messages.controls[:] 267 | clearSearchHistory() 268 | page.update() 269 | 270 | def showModelSettings(e: ControlEvent) -> None: 271 | page.go('/settings') 272 | 273 | def toggleTheme(e: ControlEvent) -> None: 274 | icon : ft.IconButton = e.control 275 | if icon.icon == ft.icons.DARK_MODE_SHARP: 276 | page.theme_mode = "dark" 277 | icon.icon = ft.icons.SUNNY 278 | user_text_field.border_color = 'white' 279 | enable_streaming.check_color = 'black' 280 | else: 281 | #icon.icon == ft.icons.DARK_MODE_OUTLINED 282 | page.theme_mode = "light" 283 | icon.icon = ft.icons.DARK_MODE_SHARP 284 | user_text_field.border_color = 'dark' 285 | enable_streaming.check_color = 'white' 286 | page.update() 287 | 288 | def view_pop(view): 289 | page.views.pop() 290 | top_view = page.views[-1] 291 | page.go(top_view.route) 292 | 293 | def showModelHub(e: ControlEvent): 294 | page.go('/model_hub') 295 | 296 | def showHistory(e: ControlEvent): 297 | page.go('/history') 298 | 299 | 300 | send_button.on_click = send 301 | clear_button.on_click = clear 302 | send_button.disabled = True 303 | clear_button.disabled = True 304 | user_text_field.on_change = enableSend 305 | model_dropdown.on_change = enableSend 306 | select_mlX_models.on_change = swapModels 307 | temp_slider.on_change = displayTemp 308 | 309 | temp_control_view = ft.Column([ 310 | temp_slider, 311 | ft.Row([ 312 | temp_label, 313 | temp_value_label 314 | ]) 315 | ]) 316 | 317 | model_control_view = ft.Row([ 318 | ft.Column([model_dropdown]), 319 | ft.Column([select_mlX_models, model_help_text], horizontal_alignment=ft.CrossAxisAlignment.START) 320 | ], alignment=ft.MainAxisAlignment.CENTER, spacing=20) 321 | 322 | controls_view = ft.Row([ 323 | temp_control_view, 324 | model_control_view 325 | ], alignment=ft.MainAxisAlignment.CENTER, spacing=20) 326 | 327 | user_input_view = ft.Row([ 328 | ft.Column([ 329 | enable_streaming, 330 | user_text_field 331 | ],alignment=ft.MainAxisAlignment.SPACE_AROUND, horizontal_alignment=ft.CrossAxisAlignment.END), 332 | ft.Column([ 333 | send_button, 334 | clear_button 335 | ], alignment=ft.MainAxisAlignment.START), 336 | ], vertical_alignment=ft.CrossAxisAlignment.CENTER, alignment=ft.MainAxisAlignment.SPACE_AROUND) 337 | 338 | top_banner_view = ft.Row([ 339 | ft.IconButton(ft.icons.SETTINGS, on_click=showModelSettings, tooltip='Expand Model Settings'), 340 | ft.Container(), 341 | ft.Row([ft.Column([ft.Row([banner_image, banner_text]), subbanner_text], alignment=ft.MainAxisAlignment.CENTER, horizontal_alignment=ft.CrossAxisAlignment.CENTER)], alignment=ft.MainAxisAlignment.CENTER), 342 | ft.Row([ 343 | ft.IconButton(ft.icons.DARK_MODE_SHARP, on_click=toggleTheme, tooltip='Toggle Dark Mode'), 344 | ft.IconButton(ft.icons.INSTALL_DESKTOP, on_click=showModelHub, tooltip='Download Models'), 345 | ft.IconButton(ft.icons.HISTORY, on_click=showHistory, tooltip='Conversation History'), 346 | #ft.PopupMenuButton() 347 | ], alignment="spacearound"), 348 | ], alignment="spacebetween") 349 | 350 | spinner_view = ft.Row([pr,pr_ph], alignment=ft.MainAxisAlignment.CENTER, vertical_alignment=ft.CrossAxisAlignment.CENTER) 351 | 352 | def route_change(e: ft.RouteChangeEvent) -> None: 353 | page.views.clear() 354 | page.views.append( 355 | ft.View( 356 | route = "/", 357 | controls=[ 358 | top_banner_view, 359 | #controls_view, 360 | tabs, 361 | spinner_view, 362 | user_input_view, 363 | #enable_streaming, 364 | ], 365 | auto_scroll=True, 366 | scroll=ft.ScrollMode.ADAPTIVE, 367 | bottom_appbar=bottom_appbar, 368 | ) 369 | ) 370 | 371 | if page.route == "/": 372 | selected_model.value = page.session.get('selected_model') 373 | selected_temp.value = page.session.get('selected_temp') 374 | selected_model_image.src = getBottomBarModelLogo(page.session.get('isMlx')) 375 | banner_image.src = getAILogo(page.session.get('isMlx')) 376 | 377 | 378 | if page.route == "/settings": 379 | page.views.append(settingsView(page)) 380 | 381 | if page.route == "/model_hub": 382 | page.views.append(modelHubView(page.theme_mode)) 383 | 384 | if page.route == "/history": 385 | page.views.append(historyView(page.theme_mode)) 386 | 387 | page.update() 388 | 389 | page.on_route_change = route_change 390 | page.on_view_pop = view_pop 391 | #page.overlay.append(controls_bottom_sheet) 392 | page.go(page.route) 393 | 394 | #page.add(top_banner_view) 395 | #page.add(controls_view) 396 | #page.add(chat_messages) 397 | #page.add(tabs) 398 | #page.add(spinner_view) 399 | #page.add(user_input_view) 400 | 401 | 402 | if __name__ == '__main__': 403 | ft.app(target=main) -------------------------------------------------------------------------------- /mlxClient.py: -------------------------------------------------------------------------------- 1 | from openai import OpenAI 2 | from typing import List 3 | 4 | class MlxClient(): 5 | 6 | def __init__(self): 7 | self.messages = [] 8 | self.client = OpenAI(base_url='http://127.0.0.1:11435/v1', api_key='pyomlx') 9 | 10 | def clear_history(self): 11 | self.messages.clear() 12 | 13 | def append_history(self, message): 14 | self.messages.append(message) 15 | 16 | def chat(self, prompt:str, model: str, temp: float, system:str = 'default') -> str: 17 | #print('Entering Chat for Mlx' + model) 18 | message = {} 19 | message['role'] = 'user' 20 | message['content'] = prompt 21 | self.messages.append(message) 22 | #data = {'model': model, 'prompt': prompt, 'temp' : temp} 23 | try: 24 | #response = requests.post(self.llmHost, data=json.dumps(data), headers={'Content-Type': 'application/json'}) 25 | #print(f'response code {response.status_code}') 26 | response = self.client.chat.completions.create(model=model, 27 | messages=self.messages) 28 | response = response.choices[0].message.content 29 | # print(response) 30 | except Exception as e: 31 | raise ValueError(e) 32 | ai_message = dict({'role' : 'assistant', 'content' : response}) 33 | self.messages.append(ai_message) 34 | return response 35 | 36 | def chat_stream(self, prompt:str, model: str, temp: float) -> str: 37 | pass 38 | 39 | def list(self) -> List[str]: 40 | response = self.client.models.list() 41 | models = [] 42 | for m in response: 43 | models.append(m.id) 44 | return models -------------------------------------------------------------------------------- /mlxLLM.py: -------------------------------------------------------------------------------- 1 | from typing import Any, List, Mapping, Optional 2 | 3 | from langchain_core.callbacks.manager import CallbackManagerForLLMRun 4 | from langchain_core.language_models.llms import LLM 5 | 6 | import requests 7 | import json 8 | 9 | class MlxLLM(LLM): 10 | llmHost = 'http://127.0.0.1:5000/serve' 11 | model = "" 12 | temp = 0.3 13 | def __init__(self, model="", temp=0.3): 14 | super().__init__() 15 | self.model = model 16 | self.temp = temp 17 | 18 | 19 | @property 20 | def _llm_type(self) -> str: 21 | return "mlxLLM" 22 | 23 | def _call( 24 | self, 25 | prompt: str, 26 | stop: Optional[List[str]] = None, 27 | run_manager: Optional[CallbackManagerForLLMRun] = None, 28 | **kwargs: Any, 29 | ) -> str: 30 | if stop is not None: 31 | raise ValueError("stop kwargs are not permitted.") 32 | data = {'model': self.model, 'prompt': prompt, 'temp' : self.temp} 33 | try: 34 | response = requests.post(self.llmHost, data=json.dumps(data), headers={'Content-Type': 'application/json'}) 35 | print(f'response code {response.status_code}') 36 | if response.status_code != 200: 37 | raise requests.ConnectionError 38 | except Exception as e: 39 | raise requests.ConnectionError 40 | return response.text 41 | 42 | @property 43 | def _identifying_params(self) -> Mapping[str, Any]: 44 | """Get the identifying parameters.""" 45 | return {"llmHost": self.llmHost, "model":self.model, "temp" : self.temp} -------------------------------------------------------------------------------- /mlxLLM_local.py: -------------------------------------------------------------------------------- 1 | from typing import Any, List, Mapping, Optional 2 | 3 | from langchain_core.callbacks.manager import CallbackManagerForLLMRun 4 | from langchain_core.language_models.llms import LLM 5 | 6 | import requests 7 | import json 8 | 9 | from mlx_lm import load, generate 10 | import os 11 | from pathlib import Path 12 | 13 | DEFAULT_HF_MLX_MODEL_REGISTRY = Path("~/.cache/huggingface/hub/").expanduser() 14 | 15 | class MlxLLM_local(LLM): 16 | model = "" 17 | temp = 0.3 18 | def __init__(self, model="", temp=0.3): 19 | super().__init__() 20 | self.model = model 21 | self.temp = temp 22 | 23 | 24 | @property 25 | def _llm_type(self) -> str: 26 | return "mlxLLM" 27 | 28 | def _call( 29 | self, 30 | prompt: str, 31 | stop: Optional[List[str]] = None, 32 | run_manager: Optional[CallbackManagerForLLMRun] = None, 33 | **kwargs: Any, 34 | ) -> str: 35 | if stop is not None: 36 | raise ValueError("stop kwargs are not permitted.") 37 | return self.firePrompt(prompt) 38 | 39 | @property 40 | def _identifying_params(self) -> Mapping[str, Any]: 41 | """Get the identifying parameters.""" 42 | return {"model":self.model, "temp" : self.temp} 43 | 44 | def firePrompt(self, prompt: str): 45 | model_dir = f'{DEFAULT_HF_MLX_MODEL_REGISTRY}/models--mlx-community--{self.model}' 46 | model_digest = "" 47 | with open(f'{model_dir}/refs/main', 'r') as f: 48 | model_digest = f.read() 49 | model_path = f'{model_dir}/snapshots/{model_digest}' 50 | model, tokenizer = load(model_path, {'trust_remote_code':True}) 51 | response = generate(model, tokenizer, prompt=prompt, max_tokens=500, temp=self.temp) 52 | return response -------------------------------------------------------------------------------- /model_hub.py: -------------------------------------------------------------------------------- 1 | import flet as ft 2 | from ollama import list 3 | from ollama import pull 4 | from ollama import delete 5 | from ollama import ProgressResponse 6 | from time import sleep 7 | 8 | model_hub_view = ft.View() 9 | 10 | settings_banner_text = ft.Text(value='Download Models', style=ft.TextStyle(font_family='CabinSketch-Bold'), size=30) 11 | settings_banner_image = ft.Image(src=f"logos/combined_logos.png", 12 | width=75, 13 | height=75, 14 | fit=ft.ImageFit.CONTAIN, 15 | ) 16 | 17 | settings_banner_view = ft.Row([ 18 | settings_banner_image, 19 | settings_banner_text 20 | ], alignment=ft.MainAxisAlignment.CENTER, vertical_alignment=ft.CrossAxisAlignment.CENTER) 21 | 22 | coming_soon_view = ft.Column([ 23 | ft.Text(value='Coming Soon!', style=ft.TextStyle(font_family='CabinSketch-Bold'), size=50) 24 | ], alignment=ft.MainAxisAlignment.CENTER, horizontal_alignment=ft.CrossAxisAlignment.CENTER) 25 | 26 | ollama_download_text = ft.Markdown(' 💡 Refer [Ollama Model Library](https://ollama.com/library) to copy the model name', auto_follow_links=True) 27 | ollama_download_help_text = ft.Markdown(' 💡 Just paste the model name, PyOllaMx will perform `ollama pull ` for you 😎', auto_follow_links=True) 28 | 29 | dlg = ft.AlertDialog() 30 | 31 | def display_alert(e): 32 | e.page.dialog = dlg 33 | dlg.title = ollama_download_help_text 34 | dlg.open = True 35 | e.page.update() 36 | 37 | def useModel(e: ft.ControlEvent): 38 | e.page.session.set('selected_model', e.control.key) 39 | e.page.dialog = dlg 40 | dlg.title = 'Done' 41 | dlg.open = True 42 | e.page.update() 43 | 44 | def use_this_model(e): 45 | print(f'Used Model is {e.control.key}') 46 | e.page.dialog = dlg 47 | dlg.title = 'Done' 48 | dlg.open = True 49 | e.page.update() 50 | 51 | def delete_this_model(e): 52 | delete(e.control.key) 53 | ollama_models_table.rows = generate_model_rows(list()) 54 | e.page.update() 55 | 56 | def try_ollama_list(): 57 | try: 58 | return list() 59 | except Exception as e: 60 | return {} 61 | 62 | def generate_model_rows(rawData: dict): 63 | rows=[] 64 | if rawData: 65 | total_models = rawData['models'] 66 | data = {} 67 | for model in total_models: 68 | data[model['name']] = model['details']['parameter_size'] 69 | 70 | for k,v in data.items(): 71 | rows.append(ft.DataRow(cells=[ 72 | ft.DataCell(ft.Text(k)), 73 | ft.DataCell(ft.Text(v)), 74 | ft.DataCell(ft.IconButton( 75 | icon=ft.icons.DELETE_FOREVER_ROUNDED, 76 | icon_color="pink600", 77 | icon_size=20, 78 | tooltip="Delete this model", 79 | key=k, 80 | on_click=delete_this_model 81 | )) 82 | ])) 83 | return rows 84 | 85 | ollama_models_table = ft.DataTable( 86 | border_radius=10, 87 | sort_column_index=0, 88 | sort_ascending=True, 89 | heading_text_style=ft.TextStyle(font_family='CabinSketch-Bold', size=15), 90 | data_row_color={"hovered": "0x30FF0000"}, 91 | show_checkbox_column=True, 92 | divider_thickness=0, 93 | columns=[ 94 | ft.DataColumn(ft.Text('Model Name')), 95 | ft.DataColumn(ft.Text('Parameters Size')), 96 | ft.DataColumn(ft.Text('')), 97 | ]) 98 | 99 | ollama_models_table.rows = generate_model_rows(try_ollama_list()) 100 | 101 | ollama_download_hint_button = ft.TextButton("more info", icon="info", on_click=display_alert) 102 | ollama_spacer_text = ft.Text('') 103 | ollama_download_textField = ft.TextField(width=500, height=50, label='Enter model name eg mistral:7b', expand=True) 104 | ollama_download_pbar = ft.ProgressBar(bar_height=20, color="green") 105 | ollama_download_pbar_text = ft.Text(max_lines=2) 106 | ollama_download_pbar.visible = False 107 | ollama_download_pbar_text.visible = False 108 | restart_required_container = ft.Text('Restart PyOllaMx to reload new models', size=15, bgcolor=ft.colors.RED_400, color=ft.colors.YELLOW_ACCENT) 109 | restart_required_container.visible = False 110 | 111 | def return_pb_value(total, current): 112 | if total == 0 or current == 0: 113 | return 0.0 114 | if total == current: 115 | return 1.0 116 | return (round((current / total), 2)) 117 | 118 | def download_from_ollama(e: ft.ControlEvent): 119 | ollama_download_pbar.visible = True 120 | ollama_download_pbar_text.visible = True 121 | model_name = ollama_download_textField.value 122 | ollama_download_pbar_text.value = f'Contacting Ollama Library to pull {model_name} ⏳ .....' 123 | e.page.update() 124 | try: 125 | status = pull(model=model_name,stream=True) 126 | for s in status: 127 | #print(f"S is {s}") 128 | if 'total' in s and 'completed' in s: 129 | #print("### Inside If") 130 | ollama_download_pbar_text.value = s['status'] 131 | ollama_download_pbar.value = return_pb_value(float(s['total']), float(s['completed'])) 132 | e.page.update() 133 | #print(f"ollama_download_pbar_text {ollama_download_pbar_text.value}") 134 | #print(f"ollama_download_pbar {ollama_download_pbar.value}") 135 | if s['status'] == 'success': 136 | ollama_download_pbar_text.value = f'Pulled {model_name} ✅ . Model list Refreshed ✅' 137 | ollama_download_pbar.value = 1.0 138 | ollama_models_table.rows = generate_model_rows(list()) 139 | restart_required_container.visible = True 140 | e.page.update() 141 | except: 142 | ollama_download_pbar.value = 1.0 143 | ollama_download_pbar_text.value = f'🚨 Error occured while pulling {model_name}. Double check model name or whether Ollama 🦙 is running 💡' 144 | e.page.update() 145 | 146 | 147 | 148 | 149 | ollama_download_button = ft.IconButton(ft.icons.DOWNLOAD_FOR_OFFLINE_ROUNDED, 150 | icon_color="green600", 151 | icon_size=60, 152 | tooltip="Download Models", 153 | on_click=download_from_ollama) 154 | ollama_available_models_text = ft.Text('Downladed models') 155 | ollama_models_list = ft.Column() 156 | 157 | mlx_download_text = ft.Text('Download MlX Model from 🤗') 158 | mlx_download_textField = ft.TextField(width=500, height=50) 159 | mlx_download_button = ft.IconButton(ft.icons.DOWNLOAD) 160 | 161 | ollama_control = ft.Column([ 162 | ft.Row([ 163 | ollama_download_text, 164 | ollama_download_hint_button 165 | ]), 166 | ft.Row([ 167 | ollama_download_textField, 168 | ollama_download_button, 169 | ], alignment=ft.MainAxisAlignment.SPACE_BETWEEN), 170 | ollama_download_pbar_text, 171 | ollama_download_pbar, 172 | restart_required_container, 173 | ft.Column([ 174 | ft.Text(), 175 | ollama_models_table 176 | ], scroll=ft.ScrollMode.ADAPTIVE, height=300), 177 | ],alignment=ft.MainAxisAlignment.START, horizontal_alignment=ft.CrossAxisAlignment.STRETCH, spacing=10) 178 | 179 | coming_soon_view = ft.Column([ 180 | ft.Text(value='Coming Soon!', style=ft.TextStyle(font_family='CabinSketch-Bold'), size=50) 181 | ], alignment=ft.MainAxisAlignment.CENTER, horizontal_alignment=ft.CrossAxisAlignment.CENTER) 182 | 183 | mlx_control = ft.Column([ 184 | mlx_download_text, 185 | ft.Row([ 186 | mlx_download_textField, 187 | mlx_download_button, 188 | ]), 189 | ], alignment=ft.MainAxisAlignment.CENTER, horizontal_alignment=ft.CrossAxisAlignment.CENTER) 190 | 191 | 192 | model_tabs = ft.Tabs( 193 | selected_index=0, 194 | animation_duration=300, 195 | tabs=[ 196 | ft.Tab( 197 | text="Ollama Models 🦙", 198 | content = ollama_control, 199 | ), 200 | ft.Tab( 201 | text="Mlx Models 🤗", 202 | content = coming_soon_view 203 | ), 204 | ], 205 | height=490, 206 | width=700, 207 | scrollable=True, 208 | ) 209 | 210 | def modelHubView(theme: str) -> ft.View: 211 | if theme == 'dark': 212 | ollama_download_textField.border_color="white" 213 | mlx_download_textField.border_color="white" 214 | else: 215 | ollama_download_textField.border_color="black" 216 | mlx_download_textField.border_color="black" 217 | 218 | model_hub_view.route = "/model_hub" 219 | model_hub_view.controls = [ 220 | ft.AppBar(title=""), 221 | settings_banner_view, 222 | #coming_soon_view, 223 | model_tabs 224 | ] 225 | return model_hub_view -------------------------------------------------------------------------------- /models.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | from typing_extensions import List 4 | import flet as ft 5 | import re 6 | from ollamaOpenAIClient import OllamaOpenAPIClient 7 | from mlxClient import MlxClient 8 | 9 | DEFAULT_OLLAMA_MODEL_REGISTRY = Path("~/.ollama/models/manifests/registry.ollama.ai/library").expanduser() 10 | DEFAULT_HF_MLX_MODEL_REGISTRY = Path("~/.cache/huggingface/hub/").expanduser() 11 | 12 | # def returnModels() -> List[str]: 13 | # #print(DEFAULT_OLLAMA_MODEL_REGISTRY) 14 | # models = [] 15 | # for d in os.listdir(DEFAULT_OLLAMA_MODEL_REGISTRY): 16 | # a_dir = os.path.join(DEFAULT_OLLAMA_MODEL_REGISTRY, d) 17 | # if os.path.isdir(a_dir): 18 | # for f in os.listdir(a_dir): 19 | # if os.path.isfile(os.path.join(a_dir, f)): 20 | # model = f'{d}:{f}' 21 | # models.append(model) 22 | # return models 23 | 24 | def returnModels() -> List[str]: 25 | print('entering retModels') 26 | models = [] 27 | try: 28 | return OllamaOpenAPIClient().list() 29 | except Exception as e: 30 | return models 31 | 32 | def returnMlxModels() -> List[str]: 33 | print('entering retModels') 34 | models = [] 35 | try: 36 | return MlxClient().list() 37 | except Exception as e: 38 | return models 39 | 40 | def retModelOptions(isMlx=False): 41 | options = [] 42 | for m in returnMlxModels() if isMlx else returnModels(): 43 | options.append(ft.dropdown.Option(m)) 44 | return options -------------------------------------------------------------------------------- /ollama/__init__.py: -------------------------------------------------------------------------------- 1 | from ollama._client import Client, AsyncClient 2 | from ollama._types import ( 3 | GenerateResponse, 4 | ChatResponse, 5 | ProgressResponse, 6 | Message, 7 | Options, 8 | RequestError, 9 | ResponseError, 10 | ) 11 | 12 | __all__ = [ 13 | 'Client', 14 | 'AsyncClient', 15 | 'GenerateResponse', 16 | 'ChatResponse', 17 | 'ProgressResponse', 18 | 'Message', 19 | 'Options', 20 | 'RequestError', 21 | 'ResponseError', 22 | 'generate', 23 | 'chat', 24 | 'embeddings', 25 | 'pull', 26 | 'push', 27 | 'create', 28 | 'delete', 29 | 'list', 30 | 'copy', 31 | 'show', 32 | ] 33 | 34 | _client = Client() 35 | 36 | generate = _client.generate 37 | chat = _client.chat 38 | embeddings = _client.embeddings 39 | pull = _client.pull 40 | push = _client.push 41 | create = _client.create 42 | delete = _client.delete 43 | list = _client.list 44 | copy = _client.copy 45 | show = _client.show 46 | -------------------------------------------------------------------------------- /ollama/_client.py: -------------------------------------------------------------------------------- 1 | import os 2 | import io 3 | import json 4 | import httpx 5 | import binascii 6 | import platform 7 | import urllib.parse 8 | from os import PathLike 9 | from pathlib import Path 10 | from hashlib import sha256 11 | from base64 import b64encode, b64decode 12 | 13 | from typing import Any, AnyStr, Union, Optional, Sequence, Mapping, Literal 14 | 15 | import sys 16 | 17 | if sys.version_info < (3, 9): 18 | from typing import Iterator, AsyncIterator 19 | else: 20 | from collections.abc import Iterator, AsyncIterator 21 | 22 | from importlib import metadata 23 | 24 | try: 25 | __version__ = metadata.version('ollama') 26 | except metadata.PackageNotFoundError: 27 | __version__ = '0.0.0' 28 | 29 | from ollama._types import Message, Options, RequestError, ResponseError 30 | 31 | 32 | class BaseClient: 33 | def __init__( 34 | self, 35 | client, 36 | host: Optional[str] = None, 37 | follow_redirects: bool = True, 38 | timeout: Any = None, 39 | **kwargs, 40 | ) -> None: 41 | """ 42 | Creates a httpx client. Default parameters are the same as those defined in httpx 43 | except for the following: 44 | - `follow_redirects`: True 45 | - `timeout`: None 46 | `kwargs` are passed to the httpx client. 47 | """ 48 | 49 | headers = kwargs.pop('headers', {}) 50 | headers['Content-Type'] = 'application/json' 51 | headers['Accept'] = 'application/json' 52 | headers['User-Agent'] = f'ollama-python/{__version__} ({platform.machine()} {platform.system().lower()}) Python/{platform.python_version()}' 53 | 54 | self._client = client( 55 | base_url=_parse_host(host or os.getenv('OLLAMA_HOST')), 56 | follow_redirects=follow_redirects, 57 | timeout=timeout, 58 | headers=headers, 59 | **kwargs, 60 | ) 61 | 62 | 63 | class Client(BaseClient): 64 | def __init__(self, host: Optional[str] = None, **kwargs) -> None: 65 | super().__init__(httpx.Client, host, **kwargs) 66 | 67 | def _request(self, method: str, url: str, **kwargs) -> httpx.Response: 68 | response = self._client.request(method, url, **kwargs) 69 | 70 | try: 71 | response.raise_for_status() 72 | except httpx.HTTPStatusError as e: 73 | raise ResponseError(e.response.text, e.response.status_code) from None 74 | 75 | return response 76 | 77 | def _stream(self, method: str, url: str, **kwargs) -> Iterator[Mapping[str, Any]]: 78 | with self._client.stream(method, url, **kwargs) as r: 79 | try: 80 | r.raise_for_status() 81 | except httpx.HTTPStatusError as e: 82 | e.response.read() 83 | raise ResponseError(e.response.text, e.response.status_code) from None 84 | 85 | for line in r.iter_lines(): 86 | partial = json.loads(line) 87 | if e := partial.get('error'): 88 | raise ResponseError(e) 89 | yield partial 90 | 91 | def _request_stream( 92 | self, 93 | *args, 94 | stream: bool = False, 95 | **kwargs, 96 | ) -> Union[Mapping[str, Any], Iterator[Mapping[str, Any]]]: 97 | return self._stream(*args, **kwargs) if stream else self._request(*args, **kwargs).json() 98 | 99 | def generate( 100 | self, 101 | model: str = '', 102 | prompt: str = '', 103 | system: str = '', 104 | template: str = '', 105 | context: Optional[Sequence[int]] = None, 106 | stream: bool = False, 107 | raw: bool = False, 108 | format: Literal['', 'json'] = '', 109 | images: Optional[Sequence[AnyStr]] = None, 110 | options: Optional[Options] = None, 111 | keep_alive: Optional[Union[float, str]] = None, 112 | ) -> Union[Mapping[str, Any], Iterator[Mapping[str, Any]]]: 113 | """ 114 | Create a response using the requested model. 115 | 116 | Raises `RequestError` if a model is not provided. 117 | 118 | Raises `ResponseError` if the request could not be fulfilled. 119 | 120 | Returns `GenerateResponse` if `stream` is `False`, otherwise returns a `GenerateResponse` generator. 121 | """ 122 | 123 | if not model: 124 | raise RequestError('must provide a model') 125 | 126 | return self._request_stream( 127 | 'POST', 128 | '/api/generate', 129 | json={ 130 | 'model': model, 131 | 'prompt': prompt, 132 | 'system': system, 133 | 'template': template, 134 | 'context': context or [], 135 | 'stream': stream, 136 | 'raw': raw, 137 | 'images': [_encode_image(image) for image in images or []], 138 | 'format': format, 139 | 'options': options or {}, 140 | 'keep_alive': keep_alive, 141 | }, 142 | stream=stream, 143 | ) 144 | 145 | def chat( 146 | self, 147 | model: str = '', 148 | messages: Optional[Sequence[Message]] = None, 149 | stream: bool = False, 150 | format: Literal['', 'json'] = '', 151 | options: Optional[Options] = None, 152 | keep_alive: Optional[Union[float, str]] = None, 153 | ) -> Union[Mapping[str, Any], Iterator[Mapping[str, Any]]]: 154 | """ 155 | Create a chat response using the requested model. 156 | 157 | Raises `RequestError` if a model is not provided. 158 | 159 | Raises `ResponseError` if the request could not be fulfilled. 160 | 161 | Returns `ChatResponse` if `stream` is `False`, otherwise returns a `ChatResponse` generator. 162 | """ 163 | 164 | if not model: 165 | raise RequestError('must provide a model') 166 | 167 | for message in messages or []: 168 | if not isinstance(message, dict): 169 | raise TypeError('messages must be a list of Message or dict-like objects') 170 | if not (role := message.get('role')) or role not in ['system', 'user', 'assistant']: 171 | raise RequestError('messages must contain a role and it must be one of "system", "user", or "assistant"') 172 | if not message.get('content'): 173 | raise RequestError('messages must contain content') 174 | if images := message.get('images'): 175 | message['images'] = [_encode_image(image) for image in images] 176 | 177 | return self._request_stream( 178 | 'POST', 179 | '/api/chat', 180 | json={ 181 | 'model': model, 182 | 'messages': messages, 183 | 'stream': stream, 184 | 'format': format, 185 | 'options': options or {}, 186 | 'keep_alive': keep_alive, 187 | }, 188 | stream=stream, 189 | ) 190 | 191 | def embeddings( 192 | self, 193 | model: str = '', 194 | prompt: str = '', 195 | options: Optional[Options] = None, 196 | keep_alive: Optional[Union[float, str]] = None, 197 | ) -> Sequence[float]: 198 | return self._request( 199 | 'POST', 200 | '/api/embeddings', 201 | json={ 202 | 'model': model, 203 | 'prompt': prompt, 204 | 'options': options or {}, 205 | 'keep_alive': keep_alive, 206 | }, 207 | ).json() 208 | 209 | def pull( 210 | self, 211 | model: str, 212 | insecure: bool = False, 213 | stream: bool = False, 214 | ) -> Union[Mapping[str, Any], Iterator[Mapping[str, Any]]]: 215 | """ 216 | Raises `ResponseError` if the request could not be fulfilled. 217 | 218 | Returns `ProgressResponse` if `stream` is `False`, otherwise returns a `ProgressResponse` generator. 219 | """ 220 | return self._request_stream( 221 | 'POST', 222 | '/api/pull', 223 | json={ 224 | 'name': model, 225 | 'insecure': insecure, 226 | 'stream': stream, 227 | }, 228 | stream=stream, 229 | ) 230 | 231 | def push( 232 | self, 233 | model: str, 234 | insecure: bool = False, 235 | stream: bool = False, 236 | ) -> Union[Mapping[str, Any], Iterator[Mapping[str, Any]]]: 237 | """ 238 | Raises `ResponseError` if the request could not be fulfilled. 239 | 240 | Returns `ProgressResponse` if `stream` is `False`, otherwise returns a `ProgressResponse` generator. 241 | """ 242 | return self._request_stream( 243 | 'POST', 244 | '/api/push', 245 | json={ 246 | 'name': model, 247 | 'insecure': insecure, 248 | 'stream': stream, 249 | }, 250 | stream=stream, 251 | ) 252 | 253 | def create( 254 | self, 255 | model: str, 256 | path: Optional[Union[str, PathLike]] = None, 257 | modelfile: Optional[str] = None, 258 | stream: bool = False, 259 | ) -> Union[Mapping[str, Any], Iterator[Mapping[str, Any]]]: 260 | """ 261 | Raises `ResponseError` if the request could not be fulfilled. 262 | 263 | Returns `ProgressResponse` if `stream` is `False`, otherwise returns a `ProgressResponse` generator. 264 | """ 265 | if (realpath := _as_path(path)) and realpath.exists(): 266 | modelfile = self._parse_modelfile(realpath.read_text(), base=realpath.parent) 267 | elif modelfile: 268 | modelfile = self._parse_modelfile(modelfile) 269 | else: 270 | raise RequestError('must provide either path or modelfile') 271 | 272 | return self._request_stream( 273 | 'POST', 274 | '/api/create', 275 | json={ 276 | 'name': model, 277 | 'modelfile': modelfile, 278 | 'stream': stream, 279 | }, 280 | stream=stream, 281 | ) 282 | 283 | def _parse_modelfile(self, modelfile: str, base: Optional[Path] = None) -> str: 284 | base = Path.cwd() if base is None else base 285 | 286 | out = io.StringIO() 287 | for line in io.StringIO(modelfile): 288 | command, _, args = line.partition(' ') 289 | if command.upper() not in ['FROM', 'ADAPTER']: 290 | print(line, end='', file=out) 291 | continue 292 | 293 | path = Path(args.strip()).expanduser() 294 | path = path if path.is_absolute() else base / path 295 | if path.exists(): 296 | args = f'@{self._create_blob(path)}\n' 297 | print(command, args, end='', file=out) 298 | 299 | return out.getvalue() 300 | 301 | def _create_blob(self, path: Union[str, Path]) -> str: 302 | sha256sum = sha256() 303 | with open(path, 'rb') as r: 304 | while True: 305 | chunk = r.read(32 * 1024) 306 | if not chunk: 307 | break 308 | sha256sum.update(chunk) 309 | 310 | digest = f'sha256:{sha256sum.hexdigest()}' 311 | 312 | try: 313 | self._request('HEAD', f'/api/blobs/{digest}') 314 | except ResponseError as e: 315 | if e.status_code != 404: 316 | raise 317 | 318 | with open(path, 'rb') as r: 319 | self._request('POST', f'/api/blobs/{digest}', content=r) 320 | 321 | return digest 322 | 323 | def delete(self, model: str) -> Mapping[str, Any]: 324 | response = self._request('DELETE', '/api/delete', json={'name': model}) 325 | return {'status': 'success' if response.status_code == 200 else 'error'} 326 | 327 | def list(self) -> Mapping[str, Any]: 328 | return self._request('GET', '/api/tags').json() 329 | 330 | def copy(self, source: str, destination: str) -> Mapping[str, Any]: 331 | response = self._request('POST', '/api/copy', json={'source': source, 'destination': destination}) 332 | return {'status': 'success' if response.status_code == 200 else 'error'} 333 | 334 | def show(self, model: str) -> Mapping[str, Any]: 335 | return self._request('POST', '/api/show', json={'name': model}).json() 336 | 337 | 338 | class AsyncClient(BaseClient): 339 | def __init__(self, host: Optional[str] = None, **kwargs) -> None: 340 | super().__init__(httpx.AsyncClient, host, **kwargs) 341 | 342 | async def _request(self, method: str, url: str, **kwargs) -> httpx.Response: 343 | response = await self._client.request(method, url, **kwargs) 344 | 345 | try: 346 | response.raise_for_status() 347 | except httpx.HTTPStatusError as e: 348 | raise ResponseError(e.response.text, e.response.status_code) from None 349 | 350 | return response 351 | 352 | async def _stream(self, method: str, url: str, **kwargs) -> AsyncIterator[Mapping[str, Any]]: 353 | async def inner(): 354 | async with self._client.stream(method, url, **kwargs) as r: 355 | try: 356 | r.raise_for_status() 357 | except httpx.HTTPStatusError as e: 358 | e.response.read() 359 | raise ResponseError(e.response.text, e.response.status_code) from None 360 | 361 | async for line in r.aiter_lines(): 362 | partial = json.loads(line) 363 | if e := partial.get('error'): 364 | raise ResponseError(e) 365 | yield partial 366 | 367 | return inner() 368 | 369 | async def _request_stream( 370 | self, 371 | *args, 372 | stream: bool = False, 373 | **kwargs, 374 | ) -> Union[Mapping[str, Any], AsyncIterator[Mapping[str, Any]]]: 375 | if stream: 376 | return await self._stream(*args, **kwargs) 377 | 378 | response = await self._request(*args, **kwargs) 379 | return response.json() 380 | 381 | async def generate( 382 | self, 383 | model: str = '', 384 | prompt: str = '', 385 | system: str = '', 386 | template: str = '', 387 | context: Optional[Sequence[int]] = None, 388 | stream: bool = False, 389 | raw: bool = False, 390 | format: Literal['', 'json'] = '', 391 | images: Optional[Sequence[AnyStr]] = None, 392 | options: Optional[Options] = None, 393 | keep_alive: Optional[Union[float, str]] = None, 394 | ) -> Union[Mapping[str, Any], AsyncIterator[Mapping[str, Any]]]: 395 | """ 396 | Create a response using the requested model. 397 | 398 | Raises `RequestError` if a model is not provided. 399 | 400 | Raises `ResponseError` if the request could not be fulfilled. 401 | 402 | Returns `GenerateResponse` if `stream` is `False`, otherwise returns an asynchronous `GenerateResponse` generator. 403 | """ 404 | if not model: 405 | raise RequestError('must provide a model') 406 | 407 | return await self._request_stream( 408 | 'POST', 409 | '/api/generate', 410 | json={ 411 | 'model': model, 412 | 'prompt': prompt, 413 | 'system': system, 414 | 'template': template, 415 | 'context': context or [], 416 | 'stream': stream, 417 | 'raw': raw, 418 | 'images': [_encode_image(image) for image in images or []], 419 | 'format': format, 420 | 'options': options or {}, 421 | 'keep_alive': keep_alive, 422 | }, 423 | stream=stream, 424 | ) 425 | 426 | async def chat( 427 | self, 428 | model: str = '', 429 | messages: Optional[Sequence[Message]] = None, 430 | stream: bool = False, 431 | format: Literal['', 'json'] = '', 432 | options: Optional[Options] = None, 433 | keep_alive: Optional[Union[float, str]] = None, 434 | ) -> Union[Mapping[str, Any], AsyncIterator[Mapping[str, Any]]]: 435 | """ 436 | Create a chat response using the requested model. 437 | 438 | Raises `RequestError` if a model is not provided. 439 | 440 | Raises `ResponseError` if the request could not be fulfilled. 441 | 442 | Returns `ChatResponse` if `stream` is `False`, otherwise returns an asynchronous `ChatResponse` generator. 443 | """ 444 | if not model: 445 | raise RequestError('must provide a model') 446 | 447 | for message in messages or []: 448 | if not isinstance(message, dict): 449 | raise TypeError('messages must be a list of strings') 450 | if not (role := message.get('role')) or role not in ['system', 'user', 'assistant']: 451 | raise RequestError('messages must contain a role and it must be one of "system", "user", or "assistant"') 452 | if not message.get('content'): 453 | raise RequestError('messages must contain content') 454 | if images := message.get('images'): 455 | message['images'] = [_encode_image(image) for image in images] 456 | 457 | return await self._request_stream( 458 | 'POST', 459 | '/api/chat', 460 | json={ 461 | 'model': model, 462 | 'messages': messages, 463 | 'stream': stream, 464 | 'format': format, 465 | 'options': options or {}, 466 | 'keep_alive': keep_alive, 467 | }, 468 | stream=stream, 469 | ) 470 | 471 | async def embeddings( 472 | self, 473 | model: str = '', 474 | prompt: str = '', 475 | options: Optional[Options] = None, 476 | keep_alive: Optional[Union[float, str]] = None, 477 | ) -> Sequence[float]: 478 | response = await self._request( 479 | 'POST', 480 | '/api/embeddings', 481 | json={ 482 | 'model': model, 483 | 'prompt': prompt, 484 | 'options': options or {}, 485 | 'keep_alive': keep_alive, 486 | }, 487 | ) 488 | 489 | return response.json() 490 | 491 | async def pull( 492 | self, 493 | model: str, 494 | insecure: bool = False, 495 | stream: bool = False, 496 | ) -> Union[Mapping[str, Any], AsyncIterator[Mapping[str, Any]]]: 497 | """ 498 | Raises `ResponseError` if the request could not be fulfilled. 499 | 500 | Returns `ProgressResponse` if `stream` is `False`, otherwise returns a `ProgressResponse` generator. 501 | """ 502 | return await self._request_stream( 503 | 'POST', 504 | '/api/pull', 505 | json={ 506 | 'name': model, 507 | 'insecure': insecure, 508 | 'stream': stream, 509 | }, 510 | stream=stream, 511 | ) 512 | 513 | async def push( 514 | self, 515 | model: str, 516 | insecure: bool = False, 517 | stream: bool = False, 518 | ) -> Union[Mapping[str, Any], AsyncIterator[Mapping[str, Any]]]: 519 | """ 520 | Raises `ResponseError` if the request could not be fulfilled. 521 | 522 | Returns `ProgressResponse` if `stream` is `False`, otherwise returns a `ProgressResponse` generator. 523 | """ 524 | return await self._request_stream( 525 | 'POST', 526 | '/api/push', 527 | json={ 528 | 'name': model, 529 | 'insecure': insecure, 530 | 'stream': stream, 531 | }, 532 | stream=stream, 533 | ) 534 | 535 | async def create( 536 | self, 537 | model: str, 538 | path: Optional[Union[str, PathLike]] = None, 539 | modelfile: Optional[str] = None, 540 | stream: bool = False, 541 | ) -> Union[Mapping[str, Any], AsyncIterator[Mapping[str, Any]]]: 542 | """ 543 | Raises `ResponseError` if the request could not be fulfilled. 544 | 545 | Returns `ProgressResponse` if `stream` is `False`, otherwise returns a `ProgressResponse` generator. 546 | """ 547 | if (realpath := _as_path(path)) and realpath.exists(): 548 | modelfile = await self._parse_modelfile(realpath.read_text(), base=realpath.parent) 549 | elif modelfile: 550 | modelfile = await self._parse_modelfile(modelfile) 551 | else: 552 | raise RequestError('must provide either path or modelfile') 553 | 554 | return await self._request_stream( 555 | 'POST', 556 | '/api/create', 557 | json={ 558 | 'name': model, 559 | 'modelfile': modelfile, 560 | 'stream': stream, 561 | }, 562 | stream=stream, 563 | ) 564 | 565 | async def _parse_modelfile(self, modelfile: str, base: Optional[Path] = None) -> str: 566 | base = Path.cwd() if base is None else base 567 | 568 | out = io.StringIO() 569 | for line in io.StringIO(modelfile): 570 | command, _, args = line.partition(' ') 571 | if command.upper() not in ['FROM', 'ADAPTER']: 572 | print(line, end='', file=out) 573 | continue 574 | 575 | path = Path(args.strip()).expanduser() 576 | path = path if path.is_absolute() else base / path 577 | if path.exists(): 578 | args = f'@{await self._create_blob(path)}\n' 579 | print(command, args, end='', file=out) 580 | 581 | return out.getvalue() 582 | 583 | async def _create_blob(self, path: Union[str, Path]) -> str: 584 | sha256sum = sha256() 585 | with open(path, 'rb') as r: 586 | while True: 587 | chunk = r.read(32 * 1024) 588 | if not chunk: 589 | break 590 | sha256sum.update(chunk) 591 | 592 | digest = f'sha256:{sha256sum.hexdigest()}' 593 | 594 | try: 595 | await self._request('HEAD', f'/api/blobs/{digest}') 596 | except ResponseError as e: 597 | if e.status_code != 404: 598 | raise 599 | 600 | async def upload_bytes(): 601 | with open(path, 'rb') as r: 602 | while True: 603 | chunk = r.read(32 * 1024) 604 | if not chunk: 605 | break 606 | yield chunk 607 | 608 | await self._request('POST', f'/api/blobs/{digest}', content=upload_bytes()) 609 | 610 | return digest 611 | 612 | async def delete(self, model: str) -> Mapping[str, Any]: 613 | response = await self._request('DELETE', '/api/delete', json={'name': model}) 614 | return {'status': 'success' if response.status_code == 200 else 'error'} 615 | 616 | async def list(self) -> Mapping[str, Any]: 617 | response = await self._request('GET', '/api/tags') 618 | return response.json() 619 | 620 | async def copy(self, source: str, destination: str) -> Mapping[str, Any]: 621 | response = await self._request('POST', '/api/copy', json={'source': source, 'destination': destination}) 622 | return {'status': 'success' if response.status_code == 200 else 'error'} 623 | 624 | async def show(self, model: str) -> Mapping[str, Any]: 625 | response = await self._request('POST', '/api/show', json={'name': model}) 626 | return response.json() 627 | 628 | 629 | def _encode_image(image) -> str: 630 | """ 631 | >>> _encode_image(b'ollama') 632 | 'b2xsYW1h' 633 | >>> _encode_image(io.BytesIO(b'ollama')) 634 | 'b2xsYW1h' 635 | >>> _encode_image('LICENSE') 636 | 'TUlUIExpY2Vuc2UKCkNvcHlyaWdodCAoYykgT2xsYW1hCgpQZXJtaXNzaW9uIGlzIGhlcmVieSBncmFudGVkLCBmcmVlIG9mIGNoYXJnZSwgdG8gYW55IHBlcnNvbiBvYnRhaW5pbmcgYSBjb3B5Cm9mIHRoaXMgc29mdHdhcmUgYW5kIGFzc29jaWF0ZWQgZG9jdW1lbnRhdGlvbiBmaWxlcyAodGhlICJTb2Z0d2FyZSIpLCB0byBkZWFsCmluIHRoZSBTb2Z0d2FyZSB3aXRob3V0IHJlc3RyaWN0aW9uLCBpbmNsdWRpbmcgd2l0aG91dCBsaW1pdGF0aW9uIHRoZSByaWdodHMKdG8gdXNlLCBjb3B5LCBtb2RpZnksIG1lcmdlLCBwdWJsaXNoLCBkaXN0cmlidXRlLCBzdWJsaWNlbnNlLCBhbmQvb3Igc2VsbApjb3BpZXMgb2YgdGhlIFNvZnR3YXJlLCBhbmQgdG8gcGVybWl0IHBlcnNvbnMgdG8gd2hvbSB0aGUgU29mdHdhcmUgaXMKZnVybmlzaGVkIHRvIGRvIHNvLCBzdWJqZWN0IHRvIHRoZSBmb2xsb3dpbmcgY29uZGl0aW9uczoKClRoZSBhYm92ZSBjb3B5cmlnaHQgbm90aWNlIGFuZCB0aGlzIHBlcm1pc3Npb24gbm90aWNlIHNoYWxsIGJlIGluY2x1ZGVkIGluIGFsbApjb3BpZXMgb3Igc3Vic3RhbnRpYWwgcG9ydGlvbnMgb2YgdGhlIFNvZnR3YXJlLgoKVEhFIFNPRlRXQVJFIElTIFBST1ZJREVEICJBUyBJUyIsIFdJVEhPVVQgV0FSUkFOVFkgT0YgQU5ZIEtJTkQsIEVYUFJFU1MgT1IKSU1QTElFRCwgSU5DTFVESU5HIEJVVCBOT1QgTElNSVRFRCBUTyBUSEUgV0FSUkFOVElFUyBPRiBNRVJDSEFOVEFCSUxJVFksCkZJVE5FU1MgRk9SIEEgUEFSVElDVUxBUiBQVVJQT1NFIEFORCBOT05JTkZSSU5HRU1FTlQuIElOIE5PIEVWRU5UIFNIQUxMIFRIRQpBVVRIT1JTIE9SIENPUFlSSUdIVCBIT0xERVJTIEJFIExJQUJMRSBGT1IgQU5ZIENMQUlNLCBEQU1BR0VTIE9SIE9USEVSCkxJQUJJTElUWSwgV0hFVEhFUiBJTiBBTiBBQ1RJT04gT0YgQ09OVFJBQ1QsIFRPUlQgT1IgT1RIRVJXSVNFLCBBUklTSU5HIEZST00sCk9VVCBPRiBPUiBJTiBDT05ORUNUSU9OIFdJVEggVEhFIFNPRlRXQVJFIE9SIFRIRSBVU0UgT1IgT1RIRVIgREVBTElOR1MgSU4gVEhFClNPRlRXQVJFLgo=' 637 | >>> _encode_image(Path('LICENSE')) 638 | 'TUlUIExpY2Vuc2UKCkNvcHlyaWdodCAoYykgT2xsYW1hCgpQZXJtaXNzaW9uIGlzIGhlcmVieSBncmFudGVkLCBmcmVlIG9mIGNoYXJnZSwgdG8gYW55IHBlcnNvbiBvYnRhaW5pbmcgYSBjb3B5Cm9mIHRoaXMgc29mdHdhcmUgYW5kIGFzc29jaWF0ZWQgZG9jdW1lbnRhdGlvbiBmaWxlcyAodGhlICJTb2Z0d2FyZSIpLCB0byBkZWFsCmluIHRoZSBTb2Z0d2FyZSB3aXRob3V0IHJlc3RyaWN0aW9uLCBpbmNsdWRpbmcgd2l0aG91dCBsaW1pdGF0aW9uIHRoZSByaWdodHMKdG8gdXNlLCBjb3B5LCBtb2RpZnksIG1lcmdlLCBwdWJsaXNoLCBkaXN0cmlidXRlLCBzdWJsaWNlbnNlLCBhbmQvb3Igc2VsbApjb3BpZXMgb2YgdGhlIFNvZnR3YXJlLCBhbmQgdG8gcGVybWl0IHBlcnNvbnMgdG8gd2hvbSB0aGUgU29mdHdhcmUgaXMKZnVybmlzaGVkIHRvIGRvIHNvLCBzdWJqZWN0IHRvIHRoZSBmb2xsb3dpbmcgY29uZGl0aW9uczoKClRoZSBhYm92ZSBjb3B5cmlnaHQgbm90aWNlIGFuZCB0aGlzIHBlcm1pc3Npb24gbm90aWNlIHNoYWxsIGJlIGluY2x1ZGVkIGluIGFsbApjb3BpZXMgb3Igc3Vic3RhbnRpYWwgcG9ydGlvbnMgb2YgdGhlIFNvZnR3YXJlLgoKVEhFIFNPRlRXQVJFIElTIFBST1ZJREVEICJBUyBJUyIsIFdJVEhPVVQgV0FSUkFOVFkgT0YgQU5ZIEtJTkQsIEVYUFJFU1MgT1IKSU1QTElFRCwgSU5DTFVESU5HIEJVVCBOT1QgTElNSVRFRCBUTyBUSEUgV0FSUkFOVElFUyBPRiBNRVJDSEFOVEFCSUxJVFksCkZJVE5FU1MgRk9SIEEgUEFSVElDVUxBUiBQVVJQT1NFIEFORCBOT05JTkZSSU5HRU1FTlQuIElOIE5PIEVWRU5UIFNIQUxMIFRIRQpBVVRIT1JTIE9SIENPUFlSSUdIVCBIT0xERVJTIEJFIExJQUJMRSBGT1IgQU5ZIENMQUlNLCBEQU1BR0VTIE9SIE9USEVSCkxJQUJJTElUWSwgV0hFVEhFUiBJTiBBTiBBQ1RJT04gT0YgQ09OVFJBQ1QsIFRPUlQgT1IgT1RIRVJXSVNFLCBBUklTSU5HIEZST00sCk9VVCBPRiBPUiBJTiBDT05ORUNUSU9OIFdJVEggVEhFIFNPRlRXQVJFIE9SIFRIRSBVU0UgT1IgT1RIRVIgREVBTElOR1MgSU4gVEhFClNPRlRXQVJFLgo=' 639 | >>> _encode_image('YWJj') 640 | 'YWJj' 641 | >>> _encode_image(b'YWJj') 642 | 'YWJj' 643 | """ 644 | 645 | if p := _as_path(image): 646 | return b64encode(p.read_bytes()).decode('utf-8') 647 | 648 | try: 649 | b64decode(image, validate=True) 650 | return image if isinstance(image, str) else image.decode('utf-8') 651 | except (binascii.Error, TypeError): 652 | ... 653 | 654 | if b := _as_bytesio(image): 655 | return b64encode(b.read()).decode('utf-8') 656 | 657 | raise RequestError('image must be bytes, path-like object, or file-like object') 658 | 659 | 660 | def _as_path(s: Optional[Union[str, PathLike]]) -> Union[Path, None]: 661 | if isinstance(s, str) or isinstance(s, Path): 662 | try: 663 | if (p := Path(s)).exists(): 664 | return p 665 | except Exception: 666 | ... 667 | return None 668 | 669 | 670 | def _as_bytesio(s: Any) -> Union[io.BytesIO, None]: 671 | if isinstance(s, io.BytesIO): 672 | return s 673 | elif isinstance(s, bytes): 674 | return io.BytesIO(s) 675 | return None 676 | 677 | 678 | def _parse_host(host: Optional[str]) -> str: 679 | """ 680 | >>> _parse_host(None) 681 | 'http://127.0.0.1:11434' 682 | >>> _parse_host('') 683 | 'http://127.0.0.1:11434' 684 | >>> _parse_host('1.2.3.4') 685 | 'http://1.2.3.4:11434' 686 | >>> _parse_host(':56789') 687 | 'http://127.0.0.1:56789' 688 | >>> _parse_host('1.2.3.4:56789') 689 | 'http://1.2.3.4:56789' 690 | >>> _parse_host('http://1.2.3.4') 691 | 'http://1.2.3.4:80' 692 | >>> _parse_host('https://1.2.3.4') 693 | 'https://1.2.3.4:443' 694 | >>> _parse_host('https://1.2.3.4:56789') 695 | 'https://1.2.3.4:56789' 696 | >>> _parse_host('example.com') 697 | 'http://example.com:11434' 698 | >>> _parse_host('example.com:56789') 699 | 'http://example.com:56789' 700 | >>> _parse_host('http://example.com') 701 | 'http://example.com:80' 702 | >>> _parse_host('https://example.com') 703 | 'https://example.com:443' 704 | >>> _parse_host('https://example.com:56789') 705 | 'https://example.com:56789' 706 | >>> _parse_host('example.com/') 707 | 'http://example.com:11434' 708 | >>> _parse_host('example.com:56789/') 709 | 'http://example.com:56789' 710 | """ 711 | 712 | host, port = host or '', 11434 713 | scheme, _, hostport = host.partition('://') 714 | if not hostport: 715 | scheme, hostport = 'http', host 716 | elif scheme == 'http': 717 | port = 80 718 | elif scheme == 'https': 719 | port = 443 720 | 721 | split = urllib.parse.urlsplit('://'.join([scheme, hostport])) 722 | host = split.hostname or '127.0.0.1' 723 | port = split.port or port 724 | 725 | return f'{scheme}://{host}:{port}' 726 | -------------------------------------------------------------------------------- /ollama/_types.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import Any, TypedDict, Sequence, Literal 3 | 4 | import sys 5 | 6 | if sys.version_info < (3, 11): 7 | from typing_extensions import NotRequired 8 | else: 9 | from typing import NotRequired 10 | 11 | 12 | class BaseGenerateResponse(TypedDict): 13 | model: str 14 | 'Model used to generate response.' 15 | 16 | created_at: str 17 | 'Time when the request was created.' 18 | 19 | done: bool 20 | 'True if response is complete, otherwise False. Useful for streaming to detect the final response.' 21 | 22 | total_duration: int 23 | 'Total duration in nanoseconds.' 24 | 25 | load_duration: int 26 | 'Load duration in nanoseconds.' 27 | 28 | prompt_eval_count: int 29 | 'Number of tokens evaluated in the prompt.' 30 | 31 | prompt_eval_duration: int 32 | 'Duration of evaluating the prompt in nanoseconds.' 33 | 34 | eval_count: int 35 | 'Number of tokens evaluated in inference.' 36 | 37 | eval_duration: int 38 | 'Duration of evaluating inference in nanoseconds.' 39 | 40 | 41 | class GenerateResponse(BaseGenerateResponse): 42 | """ 43 | Response returned by generate requests. 44 | """ 45 | 46 | response: str 47 | 'Response content. When streaming, this contains a fragment of the response.' 48 | 49 | context: Sequence[int] 50 | 'Tokenized history up to the point of the response.' 51 | 52 | 53 | class Message(TypedDict): 54 | """ 55 | Chat message. 56 | """ 57 | 58 | role: Literal['user', 'assistant', 'system'] 59 | "Assumed role of the message. Response messages always has role 'assistant'." 60 | 61 | content: str 62 | 'Content of the message. Response messages contains message fragments when streaming.' 63 | 64 | images: NotRequired[Sequence[Any]] 65 | """ 66 | Optional list of image data for multimodal models. 67 | 68 | Valid input types are: 69 | 70 | - `str` or path-like object: path to image file 71 | - `bytes` or bytes-like object: raw image data 72 | 73 | Valid image formats depend on the model. See the model card for more information. 74 | """ 75 | 76 | 77 | class ChatResponse(BaseGenerateResponse): 78 | """ 79 | Response returned by chat requests. 80 | """ 81 | 82 | message: Message 83 | 'Response message.' 84 | 85 | 86 | class ProgressResponse(TypedDict): 87 | status: str 88 | completed: int 89 | total: int 90 | digest: str 91 | 92 | 93 | class Options(TypedDict, total=False): 94 | # load time options 95 | numa: bool 96 | num_ctx: int 97 | num_batch: int 98 | num_gqa: int 99 | num_gpu: int 100 | main_gpu: int 101 | low_vram: bool 102 | f16_kv: bool 103 | logits_all: bool 104 | vocab_only: bool 105 | use_mmap: bool 106 | use_mlock: bool 107 | embedding_only: bool 108 | rope_frequency_base: float 109 | rope_frequency_scale: float 110 | num_thread: int 111 | 112 | # runtime options 113 | num_keep: int 114 | seed: int 115 | num_predict: int 116 | top_k: int 117 | top_p: float 118 | tfs_z: float 119 | typical_p: float 120 | repeat_last_n: int 121 | temperature: float 122 | repeat_penalty: float 123 | presence_penalty: float 124 | frequency_penalty: float 125 | mirostat: int 126 | mirostat_tau: float 127 | mirostat_eta: float 128 | penalize_newline: bool 129 | stop: Sequence[str] 130 | 131 | 132 | class RequestError(Exception): 133 | """ 134 | Common class for request errors. 135 | """ 136 | 137 | def __init__(self, error: str): 138 | super().__init__(error) 139 | self.error = error 140 | 'Reason for the error.' 141 | 142 | 143 | class ResponseError(Exception): 144 | """ 145 | Common class for response errors. 146 | """ 147 | 148 | def __init__(self, error: str, status_code: int = -1): 149 | try: 150 | # try to parse content as JSON and extract 'error' 151 | # fallback to raw content if JSON parsing fails 152 | error = json.loads(error).get('error', error) 153 | except json.JSONDecodeError: 154 | ... 155 | 156 | super().__init__(error) 157 | self.error = error 158 | 'Reason for the error.' 159 | 160 | self.status_code = status_code 161 | 'HTTP status code of the response.' 162 | -------------------------------------------------------------------------------- /ollamaClient.py: -------------------------------------------------------------------------------- 1 | from ollama import chat 2 | 3 | class OllamaClient(): 4 | 5 | def __init__(self): 6 | self.messages = [] 7 | 8 | def clear_history(self): 9 | self.messages.clear() 10 | 11 | def append_history(self, message): 12 | self.messages.append(message) 13 | 14 | def chat(self, prompt:str, model: str, temp: float, system:str = "default") -> str: 15 | options = dict({'temperature' : temp}) 16 | message = {} 17 | if system != 'default': 18 | sMessage = dict({'role' : 'system', 'content' : system}) 19 | self.messages.append(sMessage) 20 | message['role'] = 'user' 21 | message['content'] = prompt 22 | self.messages.append(message) 23 | response = chat(model=model, messages=self.messages, options=options) 24 | self.messages.append(response['message']) 25 | return response['message']['content'] 26 | 27 | def chat_stream(self, prompt:str, model: str, temp: float) -> str: 28 | message = {} 29 | message['role'] = 'user' 30 | message['content'] = prompt 31 | self.messages.append(message) 32 | stream = chat(model=model, messages=self.messages, options={'temperature' : temp}, stream=True) 33 | return stream 34 | 35 | 36 | if __name__ == '__main__': 37 | client = OllamaClient() 38 | while True: 39 | print('You :') 40 | response = client.chat_stream(model='dolphin-mistral:latest', temp=0.8, prompt=input()) 41 | contents = "" 42 | AiMessage = {} 43 | for chunk in response: 44 | content = chunk['message']['content'] 45 | print(content, end='', flush=True) 46 | contents += content 47 | AiMessage['role'] = 'assistant' 48 | AiMessage['content'] = contents 49 | client.append_history(AiMessage) -------------------------------------------------------------------------------- /ollamaOpenAIClient.py: -------------------------------------------------------------------------------- 1 | from openai import OpenAI 2 | from typing import List 3 | 4 | class OllamaOpenAPIClient(): 5 | 6 | def __init__(self): 7 | self.messages = [] 8 | self.client = OpenAI(base_url='http://127.0.0.1:11434/v1', api_key='pyomlx') 9 | 10 | def clear_history(self): 11 | self.messages.clear() 12 | 13 | def append_history(self, message): 14 | self.messages.append(message) 15 | 16 | def chat(self, prompt:str, model: str, temp: float, system:str = 'default') -> str: 17 | message = {} 18 | message['role'] = 'user' 19 | message['content'] = prompt 20 | self.messages.append(message) 21 | try: 22 | response = self.client.chat.completions.create(model=model, 23 | messages=self.messages) 24 | response = response.choices[0].message.content 25 | except Exception as e: 26 | raise ValueError(e) 27 | ai_message = dict({'role' : 'assistant', 'content' : response}) 28 | self.messages.append(ai_message) 29 | return response 30 | 31 | def chat_stream(self, prompt:str, model: str, temp: float) -> str: 32 | pass 33 | 34 | def list(self) -> List[str]: 35 | response = self.client.models.list() 36 | models = [] 37 | for m in response: 38 | models.append(m.id) 39 | return models 40 | 41 | 42 | if __name__ == '__main__': 43 | print(OllamaOpenAPIClient().list()) -------------------------------------------------------------------------------- /prompt.py: -------------------------------------------------------------------------------- 1 | from search import * 2 | import requests 3 | from mlxClient import MlxClient 4 | from ollamaOpenAIClient import OllamaOpenAPIClient 5 | 6 | oChatClient = OllamaOpenAPIClient() 7 | oSearchClient = OllamaOpenAPIClient() 8 | 9 | mChatClient = MlxClient() 10 | mSearchClient = MlxClient() 11 | 12 | def doOllamaChat(client, prompt: str, model: str, temp: float, system:str ='default'): 13 | response = client.chat(prompt=prompt, model=model, temp=temp, system=system) 14 | return response 15 | 16 | def doOllamaChatStream(client, prompt: str, model: str, temp: float, system:str='default'): 17 | response = client.chat_stream(prompt=prompt, model=model, temp=temp, system=system) 18 | return response 19 | 20 | def clearClientHistory(client): 21 | client.clear_history() 22 | 23 | def clearAllHistory(): 24 | oChatClient.clear_history() 25 | oSearchClient.clear_history() 26 | mChatClient.clear_history() 27 | mSearchClient.clear_history() 28 | 29 | def clearChatHistory(): 30 | clearClientHistory(oChatClient) 31 | clearClientHistory(mChatClient) 32 | 33 | def clearSearchHistory(): 34 | clearClientHistory(oSearchClient) 35 | clearClientHistory(mSearchClient) 36 | 37 | def appendHistoryforOllamaChatStream(client, message): 38 | client.append_history(message) 39 | 40 | def firePrompt(prompt: str, model: str="dolphin-mistral:latest", temp=0.4, isMlx=False, chat_mode=False, system:str = 'default') : 41 | response, keywords = "", "" 42 | if chat_mode: 43 | llm = mChatClient if isMlx else oChatClient 44 | try: 45 | response = llm.chat(model=model, temp=temp, prompt=prompt, system=system) 46 | except requests.exceptions.ConnectionError as e: 47 | err_str = f'PyOMlX' if isMlx else f'Ollama' 48 | return f'Unable to connect to {err_str}. Is it running?🤔', "" 49 | except Exception as e: 50 | return f'Generic Error Occured {e}', "" 51 | 52 | else : 53 | llm = mSearchClient if isMlx else oSearchClient 54 | # First summarize the user prompt into keywords 55 | try: 56 | keyprompt = wrapPromptWithSearch_str(prompt) 57 | keywords = llm.chat(model=model, temp=temp, prompt=keyprompt, system=getKeywordSystemPrompt()) 58 | #print(f'Keywords : {keywords}') 59 | # Do websearch with keywords 60 | search_results = doWebSearch(keywords) 61 | #print(f'Search result : {search_results}') 62 | # Do a RAG with search_results with LLM 63 | final_prompt = decorateSearchPrompt(search_results, prompt) 64 | response = llm.chat(model=model, temp=temp, prompt=final_prompt, system=getSearchSystemPrompt()) 65 | except requests.exceptions.ConnectionError as e: 66 | err_str = f'PyOMlX' if isMlx else f'Ollama' 67 | return f'Unable to connect to {err_str}. Is it running?🤔', "" 68 | except Exception as e: 69 | return f'Generic Error Occured {e}', "" 70 | 71 | return response, keywords -------------------------------------------------------------------------------- /prompts.txt: -------------------------------------------------------------------------------- 1 | system_prompt = f""" 2 | You are a helpful assistant capable of chatting with user and also perform a web research if required. Answer all questions truthfully to the best you can. If and only if you cannot answer the user_prompt, then use the information available in search_text to summarize an answer for the the user_prompt. While generating the summary, always cite the source from the given search_text by converting the citation as a clickable text using markdown format. 3 | 4 | search_text : 5 | user_prompt : 6 | """ 7 | 8 | My apple watch is constantly overheating. Are there any recent issues on this one? 9 | 10 | I'm planning a short road trip from Dallas where I'm currently. What are my options within 100 miles distance and for a budget around 2000$ ? Give me a detailed itinerary 11 | 12 | I would like a shop a ladies watch. Here are my requirements: 13 | 1) Should be under $1000 budget 14 | 2) Should be a smartwatch capable of stuffs like hear rate monitoring, exercise tracking, cycle tracking, app notifications 15 | 3) Should come with latest 1 year service support 16 | 4) Should offer free shipping within US 17 | 18 | Provide me top 3 options along with their pros and cons. -------------------------------------------------------------------------------- /pyomlx_test.py: -------------------------------------------------------------------------------- 1 | from mlx_lm import load, generate 2 | 3 | model, tokenizer = load("mlx-community/Phi-4-mini-instruct-8bit") 4 | 5 | prompt="Hi how are you?" 6 | 7 | if hasattr(tokenizer, "apply_chat_template") and tokenizer.chat_template is not None: 8 | messages = [{"role": "user", "content": prompt}] 9 | prompt = tokenizer.apply_chat_template( 10 | messages, tokenize=False, add_generation_prompt=True 11 | ) 12 | 13 | response = generate(model, tokenizer, prompt=prompt, verbose=True) 14 | print(response) 15 | -------------------------------------------------------------------------------- /reqs.txt: -------------------------------------------------------------------------------- 1 | duckduckgo_search==4.4.2 2 | flet==0.19.0 3 | flet-core==0.19.0 4 | flet-runtime==0.19.0 5 | langchain==0.1.4 6 | langchain-community==0.0.16 7 | langchain-core==0.1.17 -------------------------------------------------------------------------------- /requirements copy.txt: -------------------------------------------------------------------------------- 1 | aiohttp==3.9.3 2 | aiosignal==1.3.1 3 | altgraph==0.17.4 4 | annotated-types==0.6.0 5 | anyio==4.2.0 6 | arrow==1.3.0 7 | attrs==23.2.0 8 | binaryornot==0.4.4 9 | certifi==2023.11.17 10 | cffi==1.16.0 11 | chardet==5.2.0 12 | charset-normalizer==3.3.2 13 | click==8.1.7 14 | cookiecutter==2.5.0 15 | curl_cffi==0.6.0b9 16 | dataclasses-json==0.6.3 17 | docstring-inheritance==2.1.2 18 | duckduckgo_search==4.4.2 19 | flet==0.19.0 20 | flet-core==0.19.0 21 | flet-runtime==0.19.0 22 | frozenlist==1.4.1 23 | h11==0.14.0 24 | httpcore==0.17.3 25 | httpx==0.24.1 26 | idna==3.6 27 | Jinja2==3.1.3 28 | jsonpatch==1.33 29 | jsonpointer==2.4 30 | langchain==0.1.4 31 | langchain-community==0.0.16 32 | langchain-core==0.1.17 33 | langchainhub==0.1.14 34 | langsmith==0.0.85 35 | lxml==5.1.0 36 | macholib==1.16.3 37 | markdown-it-py==3.0.0 38 | MarkupSafe==2.1.4 39 | marshmallow==3.20.2 40 | mdurl==0.1.2 41 | modulegraph==0.19.6 42 | multidict==6.0.4 43 | mypy-extensions==1.0.0 44 | nest-asyncio==1.6.0 45 | numpy==1.26.3 46 | oauthlib==3.2.2 47 | packaging==23.2 48 | py2app==0.28.7 49 | pycparser==2.21 50 | pydantic==2.6.0 51 | pydantic_core==2.16.1 52 | Pygments==2.17.2 53 | pyinstaller==6.3.0 54 | pyinstaller-hooks-contrib==2024.0 55 | pypng==0.20220715.0 56 | python-dateutil==2.8.2 57 | python-slugify==8.0.2 58 | PyYAML==6.0.1 59 | qrcode==7.4.2 60 | repath==0.9.0 61 | requests==2.31.0 62 | rich==13.7.0 63 | setuptools==69.0.3 64 | six==1.16.0 65 | sniffio==1.3.0 66 | SQLAlchemy==2.0.25 67 | tenacity==8.2.3 68 | text-unidecode==1.3 69 | types-python-dateutil==2.8.19.20240106 70 | types-requests==2.31.0.20240125 71 | typing-inspect==0.9.0 72 | typing_extensions==4.9.0 73 | urllib3==2.2.0 74 | watchdog==3.0.0 75 | websocket-client==1.7.0 76 | websockets==11.0.3 77 | wheel==0.42.0 78 | yarl==1.9.4 79 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | duckduckgo_search==6.1.7 2 | langchain==0.1.4 3 | langchain-community==0.0.16 4 | langchain-core==0.1.17 5 | typing_extensions==4.12.2 6 | flet==0.25.2 7 | flet-core==0.24.1 8 | openai==1.58.1 9 | -------------------------------------------------------------------------------- /search.py: -------------------------------------------------------------------------------- 1 | from langchain_community.llms import Ollama 2 | from langchain_community.tools import DuckDuckGoSearchResults 3 | from langchain_community.utilities import DuckDuckGoSearchAPIWrapper 4 | 5 | wrapper = DuckDuckGoSearchAPIWrapper(time="d", max_results=3) 6 | search = DuckDuckGoSearchResults(api_wrapper=wrapper) 7 | 8 | search_prompt = f""" 9 | You are a helpful assistant capable of performing a web research. If the user_prompt is not an instruction, task or question, donot proceed to summarize. Instead, politely request the user that you cannot engage in general chat. If not, Use the information available in search_text to summarize an answer for the the user_prompt. While generating the summary, always cite the sources from the search_text by converting the citation as a hyperlink using markdown format. 10 | 11 | search_text : 12 | user_prompt : 13 | """ 14 | 15 | keyword_prompt = f""" 16 | Just summarize the input_text into few keywords. Don't respond other than the summary . 17 | """ 18 | 19 | chat_history = [] 20 | 21 | def decorateSearchPrompt(res, up): 22 | tprompt = f""" 23 | search_text : {res} 24 | user_prompt : {up} 25 | """ 26 | return tprompt 27 | 28 | def getKeywordSystemPrompt() -> str: 29 | return keyword_prompt 30 | 31 | def getSearchSystemPrompt() -> str: 32 | return search_prompt 33 | 34 | def wrapPromptWithSearch_str(search_str: str) -> str: 35 | return f'input_text : \n {search_str}' 36 | 37 | def doWebSearch(query_str:str) -> str: 38 | r = search.run(query_str) 39 | #print(f'r => {r}') 40 | return r 41 | 42 | def retSearchResults(model: str = "", search_str: str = "", temp=0.4) -> str: 43 | keywords_llm = Ollama(model=model, system=keyword_prompt, temperature=temp) 44 | query_str = keywords_llm.invoke(f'input_text : \n {search_str}') 45 | print(f'Keywords for this search texts are {query_str}') 46 | search_llm = Ollama(model=model, system=search_prompt, temperature=temp) 47 | searchResults = search.run(query_str) 48 | response = search_llm.invoke(decorateSearchPrompt(searchResults, search_str), temperature=temp) 49 | return response, query_str 50 | -------------------------------------------------------------------------------- /settings.py: -------------------------------------------------------------------------------- 1 | import flet as ft 2 | from models import * 3 | 4 | PYOLLAMX_VERSION = '0.0.5' 5 | 6 | def clearState(e: ft.ControlEvent) -> None: 7 | model_dropdown.value = "N/A" 8 | e.page.session.set('selected_model', 'N/A') 9 | e.page.session.set('isMlx', False) 10 | e.page.session.set('selected_temp', 0) 11 | 12 | def updateIsMlxInfo(e: ft.ControlEvent) -> None: 13 | clearState(e) 14 | if select_mlX_models.value: 15 | model_dropdown.options = retModelOptions(True) 16 | else: 17 | model_dropdown.options = retModelOptions() 18 | saveState(e) 19 | e.page.update() 20 | 21 | def saveState(e: ft.ControlEvent) -> None: 22 | e.page.session.set('selected_model', model_dropdown.value) 23 | e.page.session.set('isMlx', select_mlX_models.value) 24 | e.page.session.set('selected_temp', temp_slider.value) 25 | 26 | def updateTempInfo(e: ft.ControlEvent) -> None: 27 | temp_value_label.value = temp_slider.value 28 | saveState(e) 29 | e.page.update() 30 | 31 | def updateModelInfo(e: ft.ControlEvent) -> None: 32 | saveState(e) 33 | 34 | model_dropdown = ft.Dropdown( 35 | label = "Load a model^", 36 | hint_text= "Choose from available models", 37 | options = retModelOptions(), 38 | value = "unselected", 39 | dense=True, 40 | ) 41 | 42 | select_mlX_models = ft.Switch(label='Load 🖥️ mlX models from HF', 43 | value=False, 44 | adaptive=True, 45 | label_position=ft.LabelPosition.LEFT) 46 | temp_label = ft.Text(value='Temperature') 47 | temp_slider = ft.CupertinoSlider(min=0.0, max=1.0, divisions=10, value=0.3) 48 | temp_value_label = ft.Text(value=temp_slider.value) 49 | model_help_text = ft.Text('^Ollama models are loaded by default', style=ft.TextStyle(size=10), text_align=ft.TextAlign.LEFT) 50 | 51 | model_control_view = ft.Row([ 52 | ft.Column([model_dropdown]), 53 | ft.Column([select_mlX_models, model_help_text], horizontal_alignment=ft.CrossAxisAlignment.START) 54 | ], alignment=ft.MainAxisAlignment.CENTER, spacing=20) 55 | 56 | temp_control_view = ft.Column([ 57 | temp_slider, 58 | ft.Row([ 59 | temp_label, 60 | temp_value_label 61 | ]) 62 | ]) 63 | 64 | controls_view = ft.Row([ 65 | temp_control_view, 66 | model_control_view 67 | ], alignment=ft.MainAxisAlignment.CENTER, spacing=20) 68 | 69 | select_mlX_models.on_change = updateIsMlxInfo 70 | model_dropdown.on_change = updateModelInfo 71 | temp_slider.on_change = updateTempInfo 72 | 73 | settings_banner_text = ft.Text(value='PyOllaMx Settings', style=ft.TextStyle(font_family='CabinSketch-Bold'), size=30) 74 | settings_banner_image = ft.Image(src=f"logos/combined_logos.png", 75 | width=75, 76 | height=75, 77 | fit=ft.ImageFit.CONTAIN, 78 | ) 79 | settings_pyollmx_version_text = ft.Text(value=f'v{PYOLLAMX_VERSION}', style=ft.TextStyle(font_family='CabinSketch-Bold'), size=10) 80 | 81 | settings_banner_view = ft.Row([ 82 | settings_banner_image, 83 | settings_banner_text, 84 | settings_pyollmx_version_text 85 | ], alignment=ft.MainAxisAlignment.CENTER, vertical_alignment=ft.CrossAxisAlignment.CENTER) 86 | 87 | def display_model_warning_if_needed(page: ft.Page): 88 | if len(model_dropdown.options) < 1: 89 | 90 | def handle_close(e): 91 | page.close(dlg_modal) 92 | 93 | dlg_modal = ft.AlertDialog( 94 | modal=True, 95 | title=ft.Text("Unable to get model information"), 96 | content=ft.Text("Please check if Ollama / PyOMlx is running. If not, restart Ollama / PyOMlx and restart PyOllaMx as well."), 97 | actions=[ 98 | ft.TextButton("OK", on_click=handle_close), 99 | ], 100 | actions_alignment=ft.MainAxisAlignment.END, 101 | ) 102 | 103 | page.open(dlg_modal) 104 | 105 | def settingsView(page: ft.Page) -> ft.View: 106 | display_model_warning_if_needed(page) 107 | return ft.View( 108 | "/settings", 109 | controls = [ 110 | ft.AppBar(title=""), 111 | settings_banner_view, 112 | controls_view 113 | ] 114 | ) -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | from llama_index import ServiceContext 2 | from llama_index.llms import Ollama 3 | 4 | import chromadb 5 | from llama_index.vector_stores import ChromaVectorStore 6 | from llama_index import StorageContext 7 | import re 8 | 9 | 10 | 11 | def extract_urls(text): 12 | url_pattern = re.compile(r'link: (https?://[^\s,]+)') 13 | urls = re.findall(url_pattern, text) 14 | return urls 15 | 16 | from langchain_community.tools import DuckDuckGoSearchResults 17 | search = DuckDuckGoSearchResults() 18 | result = search.run("What is catch-up contribution?") 19 | results = extract_urls(result) 20 | parsed_results = [] 21 | for r in results: 22 | parsed_results.append(r.replace("]","")) 23 | 24 | print(parsed_results) 25 | 26 | chroma_client = chromadb.PersistentClient() 27 | chroma_collection = chroma_client.create_collection(name="ddg", get_or_create=True) 28 | vector_store = ChromaVectorStore(chroma_collection=chroma_collection) 29 | storage_context = StorageContext.from_defaults(vector_store=vector_store) 30 | from llama_index import VectorStoreIndex 31 | from llama_index.readers import SimpleWebPageReader 32 | documents = SimpleWebPageReader(html_to_text=True).load_data( 33 | parsed_results 34 | ) 35 | 36 | llm1 = Ollama(model="dolphin-mistral:latest") 37 | service_context = ServiceContext.from_defaults(llm=llm1, embed_model='local') 38 | 39 | index = VectorStoreIndex.from_documents( 40 | documents, storage_context=storage_context, service_context=service_context 41 | ) 42 | query_engine = index.as_query_engine() 43 | response = query_engine.query("What is catch-up contribution?") 44 | #print(response) 45 | print(response.source_nodes) 46 | 47 | -------------------------------------------------------------------------------- /test123.py: -------------------------------------------------------------------------------- 1 | from langchain_community.utilities import SearxSearchWrapper 2 | 3 | searchNxg = SearxSearchWrapper(searx_host="https://searx.sev.monster/") 4 | searchResults = searchNxg.run("What is searXNG?") 5 | print(searchResults) -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import flet as ft 2 | 3 | class Avatar(ft.UserControl): 4 | def __init__(self, type:str ='assistant'): 5 | super().__init__() 6 | self.image = 'logos/pyollama_2.jpeg' if type == 'assistant' else 'logos/vk_logo.pngs' 7 | self.ctrl = ft.CircleAvatar(foreground_image_url=self.image) 8 | 9 | def build(self): 10 | return self.ctrl 11 | 12 | class Message(): 13 | def __init__(self, user: str, text: str): 14 | self.user = user 15 | self.text = text 16 | --------------------------------------------------------------------------------