├── .gitignore ├── LICENSE ├── README.md ├── VERSION-bump.md ├── buildPyOMlx.sh ├── create-dmg.sh ├── logo.icns ├── logo.jpeg ├── logo.png ├── logo_readme.png ├── main.py ├── main.spec ├── requirements.txt ├── roadmap.md └── scripts ├── launch.sh ├── requirements.txt └── stop.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | #*.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | 162 | *.DS_Store 163 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Viz 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![](logo_readme.png) 2 | # PyOMlx 3 | ### Serve MlX models locally! 4 | 5 | ### ![Downloads](https://img.shields.io/github/downloads/kspviswa/PyOMlx/total.svg) 6 | 7 | ## Motivation 8 | Inspired by [Ollama](https://github.com/ollama/ollama) project, I wanted to have a similar experience for serving [MLX models](https://github.com/ml-explore/mlx-examples). [Mlx from ml-explore](https://github.com/ml-explore/mlx) is a new framework for running ML models in Apple Silicon. This app is intended to be used along with [PyOllaMx](https://github.com/kspviswa/pyOllaMx) 9 | 10 | I'm using these in my day to day workflow and I intend to keep develop these for my use and benifit. 11 | 12 | If you find this valuable, feel free to use it and contribute to this project as well. Please ⭐️ this repo to show your support and make my day! 13 | 14 | I'm planning on work on next items on this [roadmap.md](roadmap.md). Feel free to comment your thoughts (if any) and influence my work (if interested) 15 | 16 | MacOS DMGs are available in [Releases](https://github.com/kspviswa/PyOMlx/releases) page 17 | 18 | ## How to use 19 | 20 | 1) [Download](https://github.com/kspviswa/PyOMlx/releases) & Install the PyOMlx MacOS App 21 | 22 | 2) Run the app 23 | 24 | 3) You will now see the application running in the system tray. Use [PyOllaMx](https://github.com/kspviswa/pyOllaMx) to chat with MLX models seamlessly 25 | 26 | ## Features 27 | 28 | ### [v0.1.1](https://github.com/kspviswa/PyOMlx/releases/tag/0.1.1) 29 | - Revamped the http server portion to use the `mlx_lm.server` module. As of the latest version (`v0.20.5`) the module accepts dynamic model information from the incoming request. Hence this can be better utilized by PyOMlx. Also the `load()` function supports automatic model download from HF if not available in local `~/.cache `directory. This replaces the `/download` endpoint. 30 | - Finally, since `mlx_lm.server` runs a `httpd`, there is no need for external `flask`. So I got rid of that too. Resulting PyOMlx binary is very slim (~100 MB) and much much faster. 31 | - Rest everything is same as [v0.1.0](https://github.com/kspviswa/PyOMlx/releases/tag/0.1.0) 32 | 33 | 34 | ### [v0.1.0](https://github.com/kspviswa/PyOMlx/releases/tag/0.1.0) 35 | - Added OpenAI API Compatible [chat completions](https://platform.openai.com/docs/api-reference/chat/create) and [list models](https://platform.openai.com/docs/api-reference/models/list) endpoint. 36 | - Added `/download` endpoint to download MLX models directly from HuggingFace Hub. All models will be downloaded from [MLX Community](https://huggingface.co/mlx-community) in HF Hub. 37 | - Added `/swagger.json` endpoint to serve OpenAPI Spec of all endpoints available with PyOMlx. 38 | 39 | Now you simply use any standard OpenAI Client to interact with your MLX models easily. More info on the [v0.1.0 release](https://github.com/kspviswa/PyOMlx/releases/tag/0.1.0) page. 40 | 41 | ### [v0.0.3](https://github.com/kspviswa/PyOMlx/releases/tag/0.0.3) 42 | - Updated `mlx-lm` to support Gemma models 43 | 44 | ### [v0.0.1](https://github.com/kspviswa/PyOMlx/releases/tag/0.0.1) 45 | - Automatically discover & serve MLX models that are downloaded from [MLX Huggingface community](https://huggingface.co/mlx-community). 46 | - Easy start-up / shutdown via MacOS App 47 | - System tray indication 48 | 49 | ## Demo 50 | 51 | https://github.com/kspviswa/pyOllaMx/assets/7476271/dc686d60-182d-4f90-a771-9c1df1c70b5c -------------------------------------------------------------------------------- /VERSION-bump.md: -------------------------------------------------------------------------------- 1 | # Instructions for updating PyOMlx version 2 | 3 | Following lines needs to be touched for version bumps : 4 | 5 | - [main.py #L31](https://github.com/kspviswa/PyOMlx/blob/main/main.py#L31) 6 | - [main.spec #L49](https://github.com/kspviswa/PyOMlx/blob/main/main.spec#L49) 7 | - [launch.sh #L7](https://github.com/kspviswa/PyOMlx/blob/main/scripts/launch.sh#L7) 8 | 9 | ToDo : Need a shell automation to automatically bump these in a single command. -------------------------------------------------------------------------------- /buildPyOMlx.sh: -------------------------------------------------------------------------------- 1 | pyinstaller main.spec -------------------------------------------------------------------------------- /create-dmg.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Create a folder (named dmg) to prepare our DMG in (if it doesn't already exist). 3 | mkdir -p dist/dmg 4 | # Empty the dmg folder. 5 | rm -r dist/dmg/* 6 | # Copy the app bundle to the dmg folder. 7 | cp -r "dist/PyOMlx.app" dist/dmg 8 | # If the DMG already exists, delete it. 9 | test -f "dist/PyOMlx.dmg" && rm "dist/PyOMlx.dmg" 10 | create-dmg \ 11 | --volname "PyOMlx" \ 12 | --volicon "logo.icns" \ 13 | --window-pos 200 120 \ 14 | --window-size 600 300 \ 15 | --icon-size 100 \ 16 | --icon "PyOMlx.app" 175 120 \ 17 | --hide-extension "PyOMlx.app" \ 18 | --app-drop-link 425 120 \ 19 | "dist/PyOMlx.dmg" \ 20 | "dist/dmg/" -------------------------------------------------------------------------------- /logo.icns: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kspviswa/PyOMlx/68ce137332d92fce1410d00047a6d921dc991521/logo.icns -------------------------------------------------------------------------------- /logo.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kspviswa/PyOMlx/68ce137332d92fce1410d00047a6d921dc991521/logo.jpeg -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kspviswa/PyOMlx/68ce137332d92fce1410d00047a6d921dc991521/logo.png -------------------------------------------------------------------------------- /logo_readme.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kspviswa/PyOMlx/68ce137332d92fce1410d00047a6d921dc991521/logo_readme.png -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from PySide6.QtGui import QIcon, QAction 2 | from PySide6.QtWidgets import QApplication, QSystemTrayIcon, QMenu, QMessageBox 3 | import subprocess 4 | import os 5 | import sys 6 | 7 | basedir = os.path.dirname(__file__) 8 | script_base_path = os.path.join(basedir, "scripts") 9 | script_path = (os.path.join(basedir, "scripts", "launch.sh")) 10 | result = subprocess.run(['/bin/bash', script_path, script_base_path]) 11 | 12 | app = QApplication([]) 13 | app.setQuitOnLastWindowClosed(False) 14 | 15 | if result.returncode == 1: 16 | QMessageBox.critical( 17 | None, # Parent widget (None means it will be a top-level window) 18 | "Error", # Title of the message box 19 | "PyOMlx cannot be started due to some error. Check logs in /tmp/pyomlx-*.log", # Message text 20 | QMessageBox.Ok # Buttons to display 21 | ) 22 | sys.exit(1) 23 | 24 | def stopServer(): 25 | stop_script_path = (os.path.join(basedir, "scripts", "stop.sh")) 26 | subprocess.run(['/bin/sh', stop_script_path]) 27 | app.quit() 28 | 29 | def showAbout(): 30 | ab = QMessageBox() 31 | ab.setText("PyOMlx \n\n Version 0.1.2 \n Copyright Viswa Kumar ©️ 2025") 32 | ab.exec() 33 | 34 | # Create the icon 35 | icon = QIcon(os.path.join(basedir, "logo.png")) 36 | 37 | # Create the tray 38 | tray = QSystemTrayIcon() 39 | tray.setIcon(icon) 40 | tray.setVisible(True) 41 | 42 | # Add a Quit option to the menu. 43 | menu = QMenu() 44 | quit = QAction("Quit") 45 | about = QAction('About') 46 | quit.triggered.connect(stopServer) 47 | about.triggered.connect(showAbout) 48 | menu.addAction(about) 49 | menu.addAction(quit) 50 | 51 | # Add the menu to the tray 52 | tray.setContextMenu(menu) 53 | 54 | app.exec() -------------------------------------------------------------------------------- /main.spec: -------------------------------------------------------------------------------- 1 | # -*- mode: python ; coding: utf-8 -*- 2 | 3 | 4 | a = Analysis( 5 | ['main.py'], 6 | pathex=[], 7 | binaries=[], 8 | datas=[('logo.png', '.'), ('scripts/launch.sh', 'scripts'), ('scripts/stop.sh', 'scripts'), ('scripts/requirements.txt', 'scripts')], 9 | hiddenimports=[], 10 | hookspath=[], 11 | hooksconfig={}, 12 | runtime_hooks=[], 13 | excludes=[], 14 | noarchive=False, 15 | ) 16 | pyz = PYZ(a.pure) 17 | 18 | exe = EXE( 19 | pyz, 20 | a.scripts, 21 | [], 22 | exclude_binaries=True, 23 | name='main', 24 | debug=False, 25 | bootloader_ignore_signals=False, 26 | strip=False, 27 | upx=True, 28 | console=False, 29 | disable_windowed_traceback=False, 30 | argv_emulation=False, 31 | target_arch=None, 32 | codesign_identity=None, 33 | entitlements_file=None, 34 | ) 35 | coll = COLLECT( 36 | exe, 37 | a.binaries, 38 | a.datas, 39 | strip=False, 40 | upx=True, 41 | upx_exclude=[], 42 | name='main', 43 | ) 44 | app = BUNDLE( 45 | coll, 46 | name='PyOMlx.app', 47 | icon='logo.icns', 48 | bundle_identifier=None, 49 | version='0.1.2', 50 | info_plist={ 51 | 'NSPrincipalClass': 'NSApplication', 52 | 'NSAppleScriptEnabled': False, 53 | 'CFBundleDocumentTypes': [ 54 | { 55 | 'CFBundleTypeName': 'My File Format', 56 | 'CFBundleTypeIconFile': 'logo.icns', 57 | 'LSItemContentTypes': ['com.example.myformat'], 58 | 'LSHandlerRank': 'Owner' 59 | } 60 | ] 61 | }, 62 | ) 63 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | altgraph==0.17.4 2 | blinker==1.7.0 3 | certifi==2024.2.2 4 | charset-normalizer==3.3.2 5 | click==8.1.7 6 | filelock==3.13.1 7 | Flask==3.0.2 8 | Flask-Cors==4.0.0 9 | fsspec==2023.12.2 10 | huggingface-hub==0.27.0 11 | idna==3.6 12 | itsdangerous==2.1.2 13 | Jinja2==3.1.3 14 | macholib==1.16.3 15 | MarkupSafe==2.1.5 16 | mlx==0.21.1 17 | mlx-lm==0.20.5 18 | modulegraph==0.19.6 19 | numpy==1.26.3 20 | packaging==23.2 21 | protobuf==4.25.2 22 | py2app==0.28.7 23 | pyinstaller==6.3.0 24 | pyinstaller-hooks-contrib==2024.0 25 | PySide6==6.6.1 26 | PySide6-Addons==6.6.1 27 | PySide6-Essentials==6.6.1 28 | PyYAML==6.0.1 29 | regex==2023.12.25 30 | requests==2.31.0 31 | safetensors==0.4.2 32 | sentencepiece==0.2.0 33 | setuptools==69.0.3 34 | shiboken6==6.6.1 35 | tiktoken==0.5.2 36 | tokenizers==0.21.0 37 | tqdm==4.66.1 38 | transformers==4.47.1 39 | typing_extensions==4.9.0 40 | urllib3==2.2.0 41 | Werkzeug==3.0.1 42 | -------------------------------------------------------------------------------- /roadmap.md: -------------------------------------------------------------------------------- 1 | - [x] Basic working setup 2 | - [x] Provide a MacOS App (.dmg) file 3 | - [x] List models 4 | - [x] Support [Open AI chat_completions endpoint](https://platform.openai.com/docs/api-reference/chat/create) 5 | - [ ] Support authentication 6 | - [ ] Support Caching -------------------------------------------------------------------------------- /scripts/launch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PYOMLX_HOME="$HOME/.pyomlx" 4 | PYOMLX_VENV="$PYOMLX_HOME/.venv" 5 | PYOMLX_SERVER_PID="pid.txt" 6 | PYOMLX_VERSION_FILE="$PYOMLX_HOME/version.txt" 7 | PYOMLX_VERSION="0.1.2" # Latest PyOMlx version 8 | SCRIPT_HOME=$1 9 | PYOMLX_STARTUP_LOG="/tmp/pyomlx-startup.log" 10 | PYOMLX_LOG="/tmp/pyomlx-running.log" 11 | 12 | # Function to check if Python 3.11 is installed and return its path 13 | is_python3_11_installed() { 14 | # Define common paths to check, including Homebrew installations 15 | local paths_to_check=( 16 | "/usr/bin/python3" 17 | "/usr/local/bin/python3" 18 | "/opt/homebrew/bin/python3" 19 | "/Library/Frameworks/Python.framework/Versions/3.11/bin/python3" 20 | ) 21 | 22 | # Check each path 23 | for path in "${paths_to_check[@]}"; do 24 | if [ -x "$path" ]; then # Check if the path exists and is executable 25 | echo "$path" # Return the path if found 26 | return 0 # Return success 27 | fi 28 | done 29 | 30 | echo "" # Return an empty string if not found 31 | return 1 # Return failure 32 | } 33 | 34 | # Usage of the function in an if/else statement 35 | python_path=$(is_python3_11_installed) 36 | 37 | 38 | # Use the output in an if/else statement 39 | if [ -n "$python_path" ]; then 40 | echo "Python 3.11 is installed at: $python_path" >> $PYOMLX_LOG 41 | else 42 | echo "Python 3.11 is not installed. Due to sentencepiece requirement, PyOMlx needs Python 3.11" >> $PYOMLX_STARTUP_LOG 43 | exit 1 44 | fi 45 | 46 | install_pyomlx() { 47 | echo "Entering install" >> $PYOMLX_LOG 48 | mkdir -p $PYOMLX_HOME 49 | cd $PYOMLX_HOME 50 | echo "Inside install" >> $PYOMLX_LOG 51 | $python_path -m venv .venv 52 | . .venv/bin/activate 53 | echo "Activated env" >> $PYOMLX_LOG 54 | cp $SCRIPT_HOME/*.txt . 55 | echo "$PYOMLX_VERSION" > $PYOMLX_VERSION_FILE 56 | $PYOMLX_VENV/bin/pip install -r requirements.txt 57 | echo "Finished install" >> $PYOMLX_LOG 58 | } 59 | 60 | run_pyomlx() { 61 | cd $PYOMLX_HOME 62 | . .venv/bin/activate 63 | nohup mlx_lm.server --host 127.0.0.1 --port 11435 --use-default-chat-template > /dev/null 2>&1 & 64 | pid=$! 65 | echo $pid > $PYOMLX_SERVER_PID 66 | } 67 | 68 | #test for the presence of pyomlxhome and .venv inside 69 | if [ -d $PYOMLX_VENV ]; then 70 | echo "Directory exists. Hence re-use" >> $PYOMLX_LOG 71 | 72 | if [ -f $PYOMLX_VERSION_FILE ]; then 73 | expected_version=$(cat $PYOMLX_VERSION_FILE) 74 | if [ "$expected_version" = "$PYOMLX_VERSION" ]; then 75 | # Version is correct, proceed to activate and run server 76 | echo "Version is correct, proceed to activate and run server" >> $PYOMLX_LOG 77 | run_pyomlx 78 | else 79 | # Version is incorrect, create new version file and proceed to install requirements 80 | echo "Version is incorrect, create new version file and proceed to install requirements" >> $PYOMLX_LOG 81 | rm -rf $PYOMLX_HOME 82 | install_pyomlx 83 | run_pyomlx 84 | fi 85 | else 86 | # Version file does not exist, create it and proceed to install requirements 87 | echo "Version file does not exist, create it and proceed to install requirements" >> $PYOMLX_LOG 88 | rm -rf $PYOMLX_HOME 89 | install_pyomlx 90 | run_pyomlx 91 | fi 92 | 93 | else 94 | echo "Brand new install" >> $PYOMLX_LOG 95 | install_pyomlx 96 | run_pyomlx 97 | fi -------------------------------------------------------------------------------- /scripts/requirements.txt: -------------------------------------------------------------------------------- 1 | mlx-lm==0.21.5 2 | mlx==0.23.1 3 | tiktoken==0.8.0 -------------------------------------------------------------------------------- /scripts/stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | PYOMLX_HOME="$HOME/.pyomlx" 4 | PYOMLX_VENV="$PYOMLX_HOME/.venv" 5 | PYOMLX_SERVER_PID="pid.txt" 6 | SCRIPT_HOME=$(pwd) 7 | 8 | pid=$(cat "$PYOMLX_HOME/$PYOMLX_SERVER_PID") 9 | kill -9 $pid 10 | echo "Stopped.." 11 | 12 | --------------------------------------------------------------------------------