├── .gitignore ├── mlxserver ├── README.md ├── src │ ├── mlxserver.egg-info │ │ ├── dependency_links.txt │ │ ├── top_level.txt │ │ ├── requires.txt │ │ ├── SOURCES.txt │ │ └── PKG-INFO │ └── mlxserver │ │ ├── __init__.py │ │ ├── requirements.txt │ │ ├── load.py │ │ ├── test.py │ │ ├── show.py │ │ ├── list.py │ │ ├── pull.py │ │ ├── delete.py │ │ ├── generate.py │ │ ├── chat.py │ │ └── server.py ├── dist │ ├── mlxserver-0.1.10.tar.gz │ ├── mlxserver-0.1.12.tar.gz │ ├── mlxserver-0.1.13.tar.gz │ ├── mlxserver-0.1.10-py3-none-any.whl │ ├── mlxserver-0.1.12-py3-none-any.whl │ └── mlxserver-0.1.13-py3-none-any.whl ├── __pycache__ │ ├── chat.cpython-312.pyc │ ├── list.cpython-312.pyc │ ├── load.cpython-312.pyc │ ├── pull.cpython-312.pyc │ ├── show.cpython-312.pyc │ ├── delete.cpython-312.pyc │ ├── generate.cpython-312.pyc │ └── server.cpython-312.pyc ├── pyproject.toml └── LICENSE ├── requirements.txt ├── docs ├── .eslintrc.json ├── src │ ├── styles │ │ └── globals.css │ └── pages │ │ ├── actions │ │ ├── convert.mdx │ │ ├── list.mdx │ │ ├── show.mdx │ │ ├── pull.mdx │ │ ├── delete.mdx │ │ ├── generate.mdx │ │ └── chat.mdx │ │ ├── _meta.json │ │ ├── contribution.mdx │ │ ├── mlxserver.mdx │ │ ├── getting-started.mdx │ │ ├── models.mdx │ │ └── index.mdx ├── public │ ├── favicon.ico │ ├── image │ │ └── cheetah.png │ ├── vercel.svg │ └── next.svg ├── postcss.config.js ├── next.config.js ├── .gitignore ├── tsconfig.json ├── tailwind.config.ts ├── package.json ├── theme.config.jsx └── README.md ├── LICENSE └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /mlx-server/__pycache__ -------------------------------------------------------------------------------- /mlxserver/README.md: -------------------------------------------------------------------------------- 1 | # MLX Server 2 | -------------------------------------------------------------------------------- /mlxserver/src/mlxserver.egg-info/dependency_links.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /mlxserver/src/mlxserver.egg-info/top_level.txt: -------------------------------------------------------------------------------- 1 | mlxserver 2 | -------------------------------------------------------------------------------- /mlxserver/src/mlxserver/__init__.py: -------------------------------------------------------------------------------- 1 | from .server import MLXServer -------------------------------------------------------------------------------- /mlxserver/src/mlxserver/requirements.txt: -------------------------------------------------------------------------------- 1 | Flask 2 | mlx 3 | mlx-lm -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | mlx 2 | mlx-lm 3 | Flask 4 | requests 5 | flask_cors -------------------------------------------------------------------------------- /docs/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "next/core-web-vitals" 3 | } 4 | -------------------------------------------------------------------------------- /docs/src/styles/globals.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | -------------------------------------------------------------------------------- /docs/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mustafaaljadery/mlxserver/HEAD/docs/public/favicon.ico -------------------------------------------------------------------------------- /mlxserver/src/mlxserver.egg-info/requires.txt: -------------------------------------------------------------------------------- 1 | Flask 2 | mlx 3 | mlx-lm 4 | flask_cors 5 | huggingface_hub 6 | -------------------------------------------------------------------------------- /docs/public/image/cheetah.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mustafaaljadery/mlxserver/HEAD/docs/public/image/cheetah.png -------------------------------------------------------------------------------- /mlxserver/src/mlxserver/load.py: -------------------------------------------------------------------------------- 1 | from mlx_lm.utils import load 2 | 3 | def load_model(model): 4 | return load(model) -------------------------------------------------------------------------------- /docs/postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | }; 7 | -------------------------------------------------------------------------------- /mlxserver/src/mlxserver/test.py: -------------------------------------------------------------------------------- 1 | from server import MLXServer 2 | 3 | MLXServer("mlx-community/Nous-Hermes-2-Mistral-7B-DPO-4bit-MLX") -------------------------------------------------------------------------------- /mlxserver/dist/mlxserver-0.1.10.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mustafaaljadery/mlxserver/HEAD/mlxserver/dist/mlxserver-0.1.10.tar.gz -------------------------------------------------------------------------------- /mlxserver/dist/mlxserver-0.1.12.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mustafaaljadery/mlxserver/HEAD/mlxserver/dist/mlxserver-0.1.12.tar.gz -------------------------------------------------------------------------------- /mlxserver/dist/mlxserver-0.1.13.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mustafaaljadery/mlxserver/HEAD/mlxserver/dist/mlxserver-0.1.13.tar.gz -------------------------------------------------------------------------------- /mlxserver/src/mlxserver/show.py: -------------------------------------------------------------------------------- 1 | from huggingface_hub import ModelCard 2 | 3 | def show(model): 4 | return str(ModelCard.load(model)) -------------------------------------------------------------------------------- /mlxserver/__pycache__/chat.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mustafaaljadery/mlxserver/HEAD/mlxserver/__pycache__/chat.cpython-312.pyc -------------------------------------------------------------------------------- /mlxserver/__pycache__/list.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mustafaaljadery/mlxserver/HEAD/mlxserver/__pycache__/list.cpython-312.pyc -------------------------------------------------------------------------------- /mlxserver/__pycache__/load.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mustafaaljadery/mlxserver/HEAD/mlxserver/__pycache__/load.cpython-312.pyc -------------------------------------------------------------------------------- /mlxserver/__pycache__/pull.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mustafaaljadery/mlxserver/HEAD/mlxserver/__pycache__/pull.cpython-312.pyc -------------------------------------------------------------------------------- /mlxserver/__pycache__/show.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mustafaaljadery/mlxserver/HEAD/mlxserver/__pycache__/show.cpython-312.pyc -------------------------------------------------------------------------------- /docs/src/pages/actions/convert.mdx: -------------------------------------------------------------------------------- 1 | # Convert 2 | 3 | **Coming Soon** 4 | 5 | Convert a pytorch model to MLX format, quantize models, and more... 6 | -------------------------------------------------------------------------------- /mlxserver/__pycache__/delete.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mustafaaljadery/mlxserver/HEAD/mlxserver/__pycache__/delete.cpython-312.pyc -------------------------------------------------------------------------------- /mlxserver/__pycache__/generate.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mustafaaljadery/mlxserver/HEAD/mlxserver/__pycache__/generate.cpython-312.pyc -------------------------------------------------------------------------------- /mlxserver/__pycache__/server.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mustafaaljadery/mlxserver/HEAD/mlxserver/__pycache__/server.cpython-312.pyc -------------------------------------------------------------------------------- /mlxserver/dist/mlxserver-0.1.10-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mustafaaljadery/mlxserver/HEAD/mlxserver/dist/mlxserver-0.1.10-py3-none-any.whl -------------------------------------------------------------------------------- /mlxserver/dist/mlxserver-0.1.12-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mustafaaljadery/mlxserver/HEAD/mlxserver/dist/mlxserver-0.1.12-py3-none-any.whl -------------------------------------------------------------------------------- /mlxserver/dist/mlxserver-0.1.13-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mustafaaljadery/mlxserver/HEAD/mlxserver/dist/mlxserver-0.1.13-py3-none-any.whl -------------------------------------------------------------------------------- /docs/next.config.js: -------------------------------------------------------------------------------- 1 | const withNextra = require("nextra")({ 2 | theme: "nextra-theme-docs", 3 | themeConfig: "./theme.config.jsx", 4 | }); 5 | 6 | module.exports = withNextra(); 7 | -------------------------------------------------------------------------------- /docs/src/pages/_meta.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": "Introduction", 3 | "getting-started": "Getting Started", 4 | "models": "Models", 5 | "mlxserver": "MLXServer", 6 | "actions": "Actions", 7 | "contribution": "Contribution" 8 | } 9 | -------------------------------------------------------------------------------- /mlxserver/src/mlxserver/list.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | def list(): 4 | cache_directory = os.path.expanduser("~/.cache/huggingface/hub/") 5 | files = [file[8:] for file in os.listdir(cache_directory) if file.startswith("models--")] 6 | return files -------------------------------------------------------------------------------- /mlxserver/src/mlxserver/pull.py: -------------------------------------------------------------------------------- 1 | from huggingface_hub import snapshot_download 2 | import time 3 | 4 | def pull(model): 5 | start_time = time.time() 6 | snapshot_download(model) 7 | end_time = time.time() 8 | return {"time": f"{end_time - start_time} seconds"} 9 | -------------------------------------------------------------------------------- /docs/src/pages/actions/list.mdx: -------------------------------------------------------------------------------- 1 | # List 2 | 3 | A list of all the locally downloaded models. 4 | 5 | ## Usage 6 | 7 | ```bash copy 8 | curl -X GET 'http://127.0.0.1:5000/list' 9 | ``` 10 | 11 | ## Example 12 | 13 | **Python** 14 | 15 | ```python copy 16 | from mlxserver import MLXServer 17 | 18 | server = MLXServer(model="mlx-community/Nous-Hermes-2-Mistral-7B-DPO-4bit-MLX") 19 | ``` 20 | 21 | **curl** 22 | 23 | ```bash copy 24 | curl -X GET 'http://127.0.0.1:5000/list' 25 | ``` 26 | -------------------------------------------------------------------------------- /mlxserver/src/mlxserver/delete.py: -------------------------------------------------------------------------------- 1 | from .list import list 2 | import os 3 | 4 | def delete(model): 5 | cache_directory = os.path.expanduser("~/.cache/huggingface/hub/") 6 | files = list() 7 | 8 | if model not in files: 9 | return "Error: File not found" 10 | 11 | file_path = os.path.join(cache_directory, "models--" + model) 12 | try: 13 | os.remove(file_path) 14 | return "File removed successfully." 15 | except PermissionError: 16 | return "Permission denied. Check your file permissions." 17 | -------------------------------------------------------------------------------- /mlxserver/src/mlxserver.egg-info/SOURCES.txt: -------------------------------------------------------------------------------- 1 | LICENSE 2 | README.md 3 | pyproject.toml 4 | src/mlxserver/__init__.py 5 | src/mlxserver/chat.py 6 | src/mlxserver/delete.py 7 | src/mlxserver/generate.py 8 | src/mlxserver/list.py 9 | src/mlxserver/load.py 10 | src/mlxserver/pull.py 11 | src/mlxserver/server.py 12 | src/mlxserver/show.py 13 | src/mlxserver/test.py 14 | src/mlxserver.egg-info/PKG-INFO 15 | src/mlxserver.egg-info/SOURCES.txt 16 | src/mlxserver.egg-info/dependency_links.txt 17 | src/mlxserver.egg-info/requires.txt 18 | src/mlxserver.egg-info/top_level.txt -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | .yarn/install-state.gz 8 | 9 | # testing 10 | /coverage 11 | 12 | # next.js 13 | /.next/ 14 | /out/ 15 | 16 | # production 17 | /build 18 | 19 | # misc 20 | .DS_Store 21 | *.pem 22 | 23 | # debug 24 | npm-debug.log* 25 | yarn-debug.log* 26 | yarn-error.log* 27 | 28 | # local env files 29 | .env*.local 30 | 31 | # vercel 32 | .vercel 33 | 34 | # typescript 35 | *.tsbuildinfo 36 | next-env.d.ts 37 | -------------------------------------------------------------------------------- /docs/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["dom", "dom.iterable", "esnext"], 4 | "allowJs": true, 5 | "skipLibCheck": true, 6 | "strict": true, 7 | "noEmit": true, 8 | "esModuleInterop": true, 9 | "module": "esnext", 10 | "moduleResolution": "bundler", 11 | "resolveJsonModule": true, 12 | "isolatedModules": true, 13 | "jsx": "preserve", 14 | "incremental": true, 15 | "paths": { 16 | "@/*": ["./src/*"] 17 | } 18 | }, 19 | "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"], 20 | "exclude": ["node_modules"] 21 | } 22 | -------------------------------------------------------------------------------- /docs/src/pages/contribution.mdx: -------------------------------------------------------------------------------- 1 | # Contribution 2 | 3 | We plan to make this library the easiest way to building ML applications natively on Mac. Please contribute to the GitHub Repo, any of the following are highly appreciated: 4 | 5 | - Bug fixes are always welcome! 6 | - Highlighting of issues. 7 | - Improving test coverage (I know we need to get better at this) 8 | - Features are welcome as well, just make sure you communicate properly before building. 9 | 10 | ## Contact 11 | 12 | You can contact [Mustafa](https://twitter.com/maxaljadery/) or [Sid](https://twitter.com/siddrrsh) on Twitter/X! 13 | -------------------------------------------------------------------------------- /docs/tailwind.config.ts: -------------------------------------------------------------------------------- 1 | import type { Config } from "tailwindcss"; 2 | 3 | const config: Config = { 4 | content: [ 5 | "./src/pages/**/*.{js,ts,jsx,tsx,mdx}", 6 | "./src/components/**/*.{js,ts,jsx,tsx,mdx}", 7 | "./src/app/**/*.{js,ts,jsx,tsx,mdx}", 8 | ], 9 | theme: { 10 | extend: { 11 | backgroundImage: { 12 | "gradient-radial": "radial-gradient(var(--tw-gradient-stops))", 13 | "gradient-conic": 14 | "conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))", 15 | }, 16 | }, 17 | }, 18 | plugins: [], 19 | }; 20 | export default config; 21 | -------------------------------------------------------------------------------- /docs/public/vercel.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/src/pages/actions/show.mdx: -------------------------------------------------------------------------------- 1 | # Show 2 | 3 | Returns the details of specified model. 4 | 5 | ## Usage 6 | 7 | ```bash copy 8 | curl -X GET 'http://127.0.0.1:5000/show?model=mlx-community/Nous-Hermes-2-Mistral-7B-DPO-4bit-MLX' 9 | ``` 10 | 11 | ## Example 12 | 13 | **Python** 14 | 15 | ```python copy 16 | from mlxserver import MLXServer 17 | 18 | server = MLXServer(model="mlx-community/Nous-Hermes-2-Mistral-7B-DPO-4bit-MLX") 19 | ``` 20 | 21 | **Curl** 22 | 23 | ```bash copy 24 | curl -X GET 'http://127.0.0.1:5000/show?model=mlx-community/Nous-Hermes-2-Mistral-7B-DPO-4bit-MLX' 25 | ``` 26 | 27 | ## Parameters 28 | 29 | **model** - The model you want to show details about. 30 | -------------------------------------------------------------------------------- /docs/src/pages/actions/pull.mdx: -------------------------------------------------------------------------------- 1 | # Pull 2 | 3 | Download & store a model from huggingface locally. 4 | 5 | ## Usage 6 | 7 | ```bash copy 8 | curl -X GET 'http://127.0.0.1:5000/delete?model=mlx-community/Nous-Hermes-2-Mistral-7B-DPO-4bit-MLX' 9 | ``` 10 | 11 | ## Example 12 | 13 | **Python** 14 | 15 | ```python copy 16 | from mlxserver import MLXServer 17 | 18 | server = MLXServer(model="mlx-community/Nous-Hermes-2-Mistral-7B-DPO-4bit-MLX") 19 | ``` 20 | 21 | **Curl** 22 | 23 | ```bash copy 24 | curl -X GET 'http://127.0.0.1:5000/pull?model=mlx-community/Nous-Hermes-2-Mistral-7B-DPO-4bit-MLX' 25 | ``` 26 | 27 | ## Parameters 28 | 29 | **model** - The model you want to download. 30 | -------------------------------------------------------------------------------- /docs/src/pages/actions/delete.mdx: -------------------------------------------------------------------------------- 1 | # Delete 2 | 3 | Delete a locally stored models. 4 | 5 | ## Usage 6 | 7 | ```bash copy 8 | curl -X GET 'http://127.0.0.1:5000/delete?model=mlx-community/Nous-Hermes-2-Mistral-7B-DPO-4bit-MLX' 9 | ``` 10 | 11 | ## Example 12 | 13 | **Python** 14 | 15 | ```python copy 16 | from mlxserver import MLXServer 17 | 18 | server = MLXServer(model="mlx-community/Nous-Hermes-2-Mistral-7B-DPO-4bit-MLX") 19 | ``` 20 | 21 | **Curl** 22 | 23 | ```bash copy 24 | curl -X GET 'http://127.0.0.1:5000/delete?model=mlx-community/Nous-Hermes-2-Mistral-7B-DPO-4bit-MLX' 25 | ``` 26 | 27 | ## Parameters 28 | 29 | **model** - The model you want to delete from your local storage. 30 | -------------------------------------------------------------------------------- /docs/src/pages/mlxserver.mdx: -------------------------------------------------------------------------------- 1 | # MLXServer 2 | 3 | This is the entry point of an `mlxserver` application. It loads a model to memory and starts the server to start accepting HTTP requests. 4 | 5 | ## Usage 6 | 7 | Note: make sure you have the memory requirements to run the model. We will add more logging for this in the future. 8 | 9 | ```python copy 10 | from mlxserver import MLXServer 11 | 12 | server = MLXServer(model="mlx-community/Nous-Hermes-2-Mistral-7B-DPO-4bit-MLX") 13 | ``` 14 | 15 | ## Parameters 16 | 17 | **model** (required) - The model to load and run the server to to call. Check out [mlx-community]() for models. 18 | 19 | **port**- The port to run the server on. Default:`4431` (this is just some random port, lol) 20 | -------------------------------------------------------------------------------- /docs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "docs", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev", 7 | "build": "next build", 8 | "start": "next start", 9 | "lint": "next lint" 10 | }, 11 | "dependencies": { 12 | "next": "^14.1.0", 13 | "nextra": "^2.13.3", 14 | "nextra-theme-docs": "^2.13.3", 15 | "react": "^18.2.0", 16 | "react-dom": "^18.2.0" 17 | }, 18 | "devDependencies": { 19 | "@types/node": "^20", 20 | "@types/react": "^18", 21 | "@types/react-dom": "^18", 22 | "autoprefixer": "^10.0.1", 23 | "eslint": "^8", 24 | "eslint-config-next": "14.1.0", 25 | "postcss": "^8", 26 | "tailwindcss": "^3.3.0", 27 | "typescript": "^5" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /docs/src/pages/actions/generate.mdx: -------------------------------------------------------------------------------- 1 | # Generate 2 | 3 | Generate completion endpoint which returns a response to a prompt. 4 | 5 | ## Usage 6 | 7 | ```bash copy 8 | curl -X GET 'http://127.0.0.1:5000/generate?prompt=write%20me%20a%20poem%20about%the%20ocean&stream=true' 9 | ``` 10 | 11 | ## Example 12 | 13 | **Python** 14 | 15 | ```python copy 16 | from mlxserver import MLXServer 17 | 18 | server = MLXServer(model="mlx-community/Nous-Hermes-2-Mistral-7B-DPO-4bit-MLX") 19 | ``` 20 | 21 | **Curl** 22 | 23 | ```bash copy 24 | curl -X GET 'http://127.0.0.1:5000/generate?prompt=write%20me%20a%20poem%20about%the%20ocean&stream=true' 25 | ``` 26 | 27 | ## Parameters 28 | 29 | **propmt** (required) - The prompt you want to generate with. 30 | 31 | **stream** - Stream the output. Default: False. 32 | -------------------------------------------------------------------------------- /mlxserver/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "mlxserver" 3 | version = "0.1.14" 4 | description = "Server for the MLX ML library." 5 | readme = "README.md" 6 | license = { file = "LICENSE" } 7 | authors = [ 8 | { name = "Mustafa Aljadery", email = "aljadery@usc.edu" }, 9 | { name = "Siddharth Sharma", email = "sidshr@stanford.edu" }, 10 | ] 11 | classifiers = [ 12 | "Development Status :: 3 - Alpha", 13 | "Operating System :: OS Independent", 14 | "License :: OSI Approved :: MIT License", 15 | "Programming Language :: Python :: 3", 16 | ] 17 | requires-python = ">= 3.7" 18 | dependencies = ["Flask", "mlx", "mlx-lm", "flask_cors", "huggingface_hub"] 19 | 20 | [project.urls] 21 | Homepage = "https://www.mlxserver.com/" 22 | Repository = "https://github.com/mustafaaljadery/mlx-server/" 23 | 24 | [build-system] 25 | requires = ["setuptools>=61.0.0", "wheel", "pybind11>=2.10", "cmake>=3.24"] 26 | build-backend = "setuptools.build_meta" -------------------------------------------------------------------------------- /docs/src/pages/getting-started.mdx: -------------------------------------------------------------------------------- 1 | # Getting Started 2 | 3 | Install `mlxserver` via pip to get started. This installation will install `mlx` as well. 4 | 5 | ## Install using PyPI 6 | 7 | ```bash copy 8 | pip install mlxserver 9 | ``` 10 | 11 | To install from PyPI you must meet the following requirements: 12 | 13 | - Using an M series chip (Apple silicon) 14 | - Using a native Python >= 3.8 15 | - macOS >= 13.3 16 | 17 | ## Usage 18 | 19 | The following is an example of using `Mistral 7B Nous Hermes 2` for generating text: 20 | 21 | **Python** 22 | 23 | ```python copy 24 | from mlxserver import MLXServer 25 | 26 | server = MLXServer(model="mlx-community/Nous-Hermes-2-Mistral-7B-DPO-4bit-MLX") 27 | ``` 28 | 29 | **Curl** 30 | 31 | ```bash copy 32 | curl -X GET 'http://127.0.0.1:5000/generate?prompt=write%20me%20a%20poem%20about%the%20ocean&stream=true' 33 | ``` 34 | 35 | ## Note 36 | 37 | This library only runs on Apple Metal. The MLX library focuses on Apple Metal acceleration. 38 | -------------------------------------------------------------------------------- /docs/src/pages/models.mdx: -------------------------------------------------------------------------------- 1 | # Models 2 | 3 | To view all of the models, check out `mlx-lm` on [Huggingface](https://huggingface.co/mlx-community). Our MLX server uses all of the models supported by the [mlx-community](https://huggingface.co/mlx-community). 4 | 5 | ## Popular Models 6 | 7 | Here's a list of some of the most popular models and their usage: 8 | 9 | #### Mistral 7B Instruct Nous Research 10 | 11 | ```python copy 12 | from mlxserver import MLXServer 13 | 14 | server = MLXServer(model="mlx-community/Nous-Hermes-2-Mistral-7B-DPO-4bit-MLX") 15 | ``` 16 | 17 | #### Mixtral Instruct 18 | 19 | ```python copy 20 | from mlxserver import MLXServer 21 | 22 | server = MLXServer(model="mlx-community/Mixtral-8x7B-Instruct-v0.1") 23 | ``` 24 | 25 | #### Gemma 26 | 27 | ```python copy 28 | from mlxserver import MLXServer 29 | 30 | server = MLXServer(model="mlx-community/quantized-gemma-7b-it") 31 | ``` 32 | 33 | #### CodeLlama 7B Instruct 34 | 35 | ```python copy 36 | from mlxserver import MLXServer 37 | 38 | server = MLXServer(model="mlx-community/CodeLlama-7b-mlx") 39 | ``` 40 | -------------------------------------------------------------------------------- /mlxserver/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2023 Mustafa Aljadery, Siddharth Sharma. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /docs/src/pages/actions/chat.mdx: -------------------------------------------------------------------------------- 1 | # Chat 2 | 3 | Chat completion endpoint to chat with a model. 4 | 5 | ## Usage 6 | 7 | ```bash copy 8 | curl -X POST -H "Content-Type: application/json" -d '{ 9 | "messages": [ 10 | { 11 | "role": "user", 12 | "content": "Write me a poem about the ocean." 13 | } 14 | ], 15 | "stream": "true" 16 | }' http://127.0.0.1:5000/chat 17 | ``` 18 | 19 | ## Example 20 | 21 | **Python** 22 | 23 | ```python 24 | from mlxserver import MLXServer 25 | 26 | server = MLXServer(model="mlx-community/Nous-Hermes-2-Mistral-7B-DPO-4bit-MLX") 27 | ``` 28 | 29 | **Curl** 30 | 31 | ```bash copy 32 | curl -X POST -H "Content-Type: application/json" -d '{ 33 | "messages": [ 34 | { 35 | "role": "user", 36 | "content": "Write me a poem about the ocean." 37 | } 38 | ], 39 | "stream": "true" 40 | }' http://127.0.0.1:5000/chat 41 | ``` 42 | 43 | ## Parameters 44 | 45 | **messages** - In the format of a dict with `role` and `content`. 46 | 47 | **stream** - Stream the output. Default: False. 48 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Mustafa Aljadery 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /docs/theme.config.jsx: -------------------------------------------------------------------------------- 1 | export default { 2 | logo: ( 3 |

8 | MLX Server 9 |

10 | ), 11 | project: { 12 | link: "https://github.com/mustafaaljadery/mlx-server", 13 | }, 14 | useNextSeoProps() { 15 | return { 16 | titleTemplate: "%s – MLX Server", 17 | }; 18 | }, 19 | head: ( 20 | <> 21 | 22 | 23 | 24 | 25 | ), 26 | primaryHue: 301, 27 | primarySaturation: 33, 28 | footer: { 29 | text: ( 30 | 35 | MIT {new Date().getFullYear()} ©{" "} 36 | 37 | Mustafa Aljadery 38 | {" "} 39 | &{" "} 40 | 41 | Siddharth Sharma 42 | 43 | . 44 | 45 | ), 46 | }, 47 | docsRepositoryBase: 48 | 'https://github.com/mustafaaljadery/mlx-server/blob/main/docs' 49 | }; 50 | -------------------------------------------------------------------------------- /docs/src/pages/index.mdx: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | This Python library is the easist way to begin building on top of Apple's machine learning library, `MLX`. 4 | 5 | #### Quick Example: 6 | 7 | **Python** 8 | 9 | ```python copy 10 | from mlxserver import MLXServer 11 | 12 | server = MLXServer(model="mlx-community/Nous-Hermes-2-Mistral-7B-DPO-4bit-MLX") 13 | ``` 14 | 15 | **Curl** 16 | 17 | ```bash copy 18 | curl -X GET 'http://127.0.0.1:5000/generate?prompt=write%20me%20a%20poem%20about%the%20ocean&stream=true' 19 | ``` 20 | 21 | [Get Started](/getting-started) 22 | 23 | ## Goals 24 | 25 | This library's goal is to make it as easy as possible to build ML applications on top of the MLX library. The lower the barrier to entry for building in ML, the more great applications will be built. 26 | 27 | #### Why `MLX`? 28 | 29 | ![Picture of a cheetah](/image/cheetah.png) 30 | 31 | Because it's the fastest on Apple Silicon and it's going to be the best maintained for a while. 32 | 33 | ## Credits & contributions 34 | 35 | This library was created by [Mustafa Aljadery](https://www.maxaljadery.com/) and [Siddharth Sharma](https://stanford.edu/~sidshr/). 36 | 37 | If you would like to contribute to this library please visit the [GitHub Repo](https://github.com/mustafaaljadery/mlx-server/). 38 | -------------------------------------------------------------------------------- /docs/public/next.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MLX Server 2 | 3 | The easiest way to create a server to inference MLX models. 4 | 5 | ## Documentation 6 | 7 | For documentation and guides, visit [mlxserver.com](https://mlxserver.com). 8 | 9 | View all of the support models [here](https://huggingface.co/mlx-community). 10 | 11 | ## Getting Started 12 | 13 | Install `mlxserver` via pip to get started. This installation will install `mlx` as well. 14 | 15 | ### Install using PyPI 16 | 17 | ```bash copy 18 | pip install mlxserver 19 | ``` 20 | 21 | To install from PyPI you must meet the following requirements: 22 | 23 | - Using an M series chip (Apple silicon) 24 | - Using a native Python >= 3.8 25 | - macOS >= 13.5 26 | 27 | ### Usage 28 | 29 | The following is an example of using `Mistral 7B Nous Hermes 2` for generating text: 30 | 31 | **Python** 32 | 33 | ```python copy 34 | from mlxserver import MLXServer 35 | 36 | server = MLXServer(model="mlx-community/Nous-Hermes-2-Mistral-7B-DPO-4bit-MLX") 37 | ``` 38 | 39 | **Curl** 40 | 41 | ```bash copy 42 | curl -X GET 'http://127.0.0.1:5000/generate?prompt=write%20me%20a%20poem%20about%the%20ocean&stream=true' 43 | ``` 44 | 45 | ### Note 46 | 47 | This library only runs on Apple Metal. The MLX library focuses on Apple Metal acceleration. 48 | 49 | ## Authors 50 | 51 | This library was made by [Mustafa Aljadery](https://www.maxaljadery.com/) & [Siddharth Sharma](https://stanford.edu/~sidshr/). 52 | -------------------------------------------------------------------------------- /mlxserver/src/mlxserver/generate.py: -------------------------------------------------------------------------------- 1 | from mlx_lm.utils import load, generate_step 2 | import mlx.core as mx 3 | import re 4 | 5 | temperature = 0 6 | context_length = 1000 7 | stop_words = ["<|im_start|>", "<|im_end|>", "", ""] 8 | 9 | def generate_steps(the_prompt, the_model, tokenizer): 10 | tokens = [] 11 | skip = 0 12 | 13 | for (token, prob), n in zip(generate_step(mx.array(tokenizer.encode(the_prompt)), the_model, temperature), 14 | range(context_length)): 15 | try: 16 | if token == tokenizer.eos_token_id: 17 | break 18 | 19 | tokens.append(token.item()) 20 | text = tokenizer.decode(tokens) 21 | 22 | trim = None 23 | 24 | for sw in stop_words: 25 | if text[-len(sw):].lower() == sw: 26 | return 27 | else: 28 | for i, _ in enumerate(sw, start=1): 29 | if text[-i:].lower() == sw[:i]: 30 | trim = -i 31 | 32 | yield text[skip:trim] 33 | skip = len(text) 34 | except: 35 | continue 36 | 37 | def generate(prompt, model, tokenizer): 38 | response = '' 39 | 40 | for chunk in generate_steps(prompt, model, tokenizer): 41 | try: 42 | response = response + chunk 43 | if True: 44 | response = re.sub(r"^/\*+/", "", response) 45 | response = re.sub(r"^:+", "", response) 46 | response = response.replace('�', '') 47 | except: 48 | continue 49 | 50 | return response -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app). 2 | 3 | ## Getting Started 4 | 5 | First, run the development server: 6 | 7 | ```bash 8 | npm run dev 9 | # or 10 | yarn dev 11 | # or 12 | pnpm dev 13 | # or 14 | bun dev 15 | ``` 16 | 17 | Open [http://localhost:3000](http://localhost:3000) with your browser to see the result. 18 | 19 | You can start editing the page by modifying `pages/index.tsx`. The page auto-updates as you edit the file. 20 | 21 | [API routes](https://nextjs.org/docs/api-routes/introduction) can be accessed on [http://localhost:3000/api/hello](http://localhost:3000/api/hello). This endpoint can be edited in `pages/api/hello.ts`. 22 | 23 | The `pages/api` directory is mapped to `/api/*`. Files in this directory are treated as [API routes](https://nextjs.org/docs/api-routes/introduction) instead of React pages. 24 | 25 | This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font. 26 | 27 | ## Learn More 28 | 29 | To learn more about Next.js, take a look at the following resources: 30 | 31 | - [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API. 32 | - [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial. 33 | 34 | You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome! 35 | 36 | ## Deploy on Vercel 37 | 38 | The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js. 39 | 40 | Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details. 41 | -------------------------------------------------------------------------------- /mlxserver/src/mlxserver.egg-info/PKG-INFO: -------------------------------------------------------------------------------- 1 | Metadata-Version: 2.1 2 | Name: mlxserver 3 | Version: 0.1.14 4 | Summary: Server for the MLX ML library. 5 | Author-email: Mustafa Aljadery , Siddharth Sharma 6 | License: Copyright 2023 Mustafa Aljadery, Siddharth Sharma. 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 11 | 12 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 13 | Project-URL: Homepage, https://www.mlxserver.com/ 14 | Project-URL: Repository, https://github.com/mustafaaljadery/mlx-server/ 15 | Classifier: Development Status :: 3 - Alpha 16 | Classifier: Operating System :: OS Independent 17 | Classifier: License :: OSI Approved :: MIT License 18 | Classifier: Programming Language :: Python :: 3 19 | Requires-Python: >=3.7 20 | Description-Content-Type: text/markdown 21 | License-File: LICENSE 22 | Requires-Dist: Flask 23 | Requires-Dist: mlx 24 | Requires-Dist: mlx-lm 25 | Requires-Dist: flask_cors 26 | Requires-Dist: huggingface_hub 27 | 28 | # MLX Server 29 | -------------------------------------------------------------------------------- /mlxserver/src/mlxserver/chat.py: -------------------------------------------------------------------------------- 1 | from mlx_lm.utils import load, generate_step 2 | import mlx.core as mx 3 | import re 4 | 5 | temperature = 0 6 | context_length = 1000 7 | stop_words = ["<|im_start|>", "<|im_end|>", "", ""] 8 | 9 | def generate_chat_steps(the_prompt, the_model, tokenizer): 10 | tokens = [] 11 | skip = 0 12 | 13 | for (token, prob), n in zip(generate_step(mx.array(tokenizer.encode(the_prompt)), the_model, temperature), 14 | range(context_length)): 15 | 16 | if token == tokenizer.eos_token_id: 17 | break 18 | 19 | tokens.append(token.item()) 20 | text = tokenizer.decode(tokens) 21 | 22 | trim = None 23 | 24 | for sw in stop_words: 25 | if text[-len(sw):].lower() == sw: 26 | return 27 | else: 28 | for i, _ in enumerate(sw, start=1): 29 | if text[-i:].lower() == sw[:i]: 30 | trim = -i 31 | 32 | yield text[skip:trim] 33 | skip = len(text) 34 | 35 | def convert_chat(messages, role_mapping = None): 36 | default_role_mapping = { 37 | "system_prompt": "A chat between a curious user and an artificial intelligence assistant. The assistant follows the given rules no matter what.", 38 | "system": "ASSISTANT's RULE: ", 39 | "user": "USER: ", 40 | "assistant": "ASSISTANT: ", 41 | "stop": "\n", 42 | } 43 | role_mapping = role_mapping if role_mapping is not None else default_role_mapping 44 | 45 | prompt = "" 46 | for line in messages: 47 | role_prefix = role_mapping.get(line["role"], "") 48 | stop = role_mapping.get("stop", "") 49 | content = line.get("content", "") 50 | prompt += f"{role_prefix}{content}{stop}" 51 | 52 | prompt += role_mapping.get("assistant", "") 53 | return prompt.rstrip() 54 | 55 | def chat(messages, model, tokenizer): 56 | prompt = convert_chat(messages) 57 | response = '' 58 | 59 | for chunk in generate_chat_steps(prompt, model, tokenizer): 60 | response = response + chunk 61 | if True: 62 | response = re.sub(r"^/\*+/", "", response) 63 | response = re.sub(r"^:+", "", response) 64 | response = response.replace('�', '') 65 | 66 | return response 67 | -------------------------------------------------------------------------------- /mlxserver/src/mlxserver/server.py: -------------------------------------------------------------------------------- 1 | from flask import Flask, request, Response, stream_with_context 2 | from flask_cors import CORS 3 | from .generate import generate, generate_steps 4 | from .load import load_model 5 | from .pull import pull 6 | from .list import list 7 | from .show import show 8 | from .delete import delete 9 | from .chat import chat, convert_chat 10 | 11 | class MLXServer(): 12 | def __init__(self, model, port = 5000): 13 | if model is None: 14 | raise ValueError("Model cannot be blank") 15 | 16 | self.app = Flask(__name__) 17 | CORS(self.app) 18 | self.model = model 19 | self.port = port 20 | self.loaded_model, self.tokenizer = load_model(model) 21 | 22 | @self.app.route('/generate') 23 | def generate_endpoint(): 24 | prompt = request.args.get("prompt") 25 | stream = request.args.get('stream') == 'true' 26 | if stream: 27 | return Response(stream_with_context(generate_steps(prompt, self.loaded_model, self.tokenizer))) 28 | else: 29 | return generate(prompt, self.loaded_model, self.tokenizer) 30 | 31 | @self.app.route("/chat", methods=['POST']) 32 | def chat_endpoint(): 33 | messages = request.json.get("messages") 34 | stream = request.json.get('stream') == 'true' 35 | if stream: 36 | prompt = convert_chat(messages) 37 | return Response(stream_with_context(generate_steps(prompt, self.loaded_model, self.tokenizer))) 38 | else: 39 | return chat(messages, self.loaded_model, self.tokenizer) 40 | 41 | @self.app.route("/list") 42 | def list_endpiont(): 43 | return list() 44 | 45 | @self.app.route("/convert") 46 | def convert_endpoint(): 47 | return "coming soon :)" 48 | 49 | @self.app.route("/delete") 50 | def delete_endpoint(): 51 | model = request.args.get("model") 52 | return delete(model) 53 | 54 | @self.app.route("/pull") 55 | def pull_endpoint(): 56 | model = request.args.get('model') 57 | return pull(model) 58 | 59 | @self.app.route("/show") 60 | def show_endpoint(): 61 | model = request.args.get('model') 62 | return show(model) 63 | 64 | self.app.run(port=self.port) --------------------------------------------------------------------------------