├── .gitignore ├── requirements.txt ├── pyproject.toml ├── LICENSE ├── benchmarks.py ├── README.md └── benchmarks ├── 1080TI ├── benchmarks_eval.csv ├── benchmarks_prompt_eval.csv └── plot_prompt_eval.svg ├── M1MAX ├── benchmarks_eval.csv ├── benchmarks_prompt_eval.csv ├── plot_eval.svg └── plot_prompt_eval.svg └── A6000 ├── benchmarks_eval.csv └── benchmarks_prompt_eval.csv /.gitignore: -------------------------------------------------------------------------------- 1 | .venv 2 | .DS_Store 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | ollama 2 | pandas 3 | seaborn 4 | matplotlib 5 | tqdm 6 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.ruff] 2 | line-length = 88 3 | indent-width = 4 4 | task-tags = ["FIX", "TODO", "HACK", "WARN", "PERF", "NOTE", "TEST"] 5 | 6 | [tool.ruff.format] 7 | quote-style = "double" 8 | 9 | [tool.pyright] 10 | typeCheckingMode = "off" 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 S1M0N38 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /benchmarks.py: -------------------------------------------------------------------------------- 1 | import ollama 2 | import json 3 | import pandas as pd 4 | import seaborn as sns 5 | import matplotlib.pyplot as plt 6 | from tqdm import tqdm 7 | from pathlib import Path 8 | 9 | 10 | MESSAGE_LIMIT = None 11 | 12 | # # A6000 13 | # MODELS = [ 14 | # # Small 15 | # "llama3:8b-instruct-q4_0", 16 | # "gemma:7b-instruct-v1.1-q4_0", 17 | # "mistral:7b-instruct-v0.2-q4_0", 18 | # "wizardlm2:7b-q4_0", 19 | # # Medium 20 | # "llama3:70b-instruct-q4_0", 21 | # "command-r:35b-v0.1-q4_0", 22 | # "mixtral:8x7b-instruct-v0.1-q4_0", 23 | # # Large 24 | # # add here large models to test ... 25 | # ] 26 | 27 | # M1 MAX and 1080TI 28 | MODELS = [ 29 | # Tiny 30 | "phi3:3.8b-mini-instruct-4k-q4_K_M", 31 | # Small 32 | "llama3:8b-instruct-q4_0", 33 | "gemma:7b-instruct-v1.1-q4_0", 34 | "mistral:7b-instruct-v0.2-q4_0", 35 | "wizardlm2:7b-q4_0", 36 | ] 37 | 38 | 39 | AVAILABLE_MODELS = [model["model"] for model in ollama.list()["models"]] 40 | [ollama.pull(model) for model in MODELS if model not in AVAILABLE_MODELS] 41 | 42 | COLUMNS = [ 43 | "model", 44 | "total_duration", 45 | "load_duration", 46 | "prompt_eval_count", 47 | "prompt_eval_duration", 48 | "eval_count", 49 | "eval_duration", 50 | ] 51 | 52 | 53 | # NOTE: Change the path to the benchmarks folder 54 | BENCHMARKS_PATH = Path("benchmarks") / "1080TI" 55 | EVAL_PATH = BENCHMARKS_PATH / "benchmarks_eval.csv" 56 | PROMPT_EVAL_PATH = BENCHMARKS_PATH / "benchmarks_prompt_eval.csv" 57 | 58 | with open(BENCHMARKS_PATH.parent / "messages.json") as f: 59 | MESSAGES = json.load(f) 60 | 61 | 62 | def benchmark_eval(model: str, df: pd.DataFrame) -> None: 63 | messages = MESSAGES["eval_messages"][:MESSAGE_LIMIT] 64 | for message in tqdm(messages): 65 | df.loc[len(df)] = ollama.chat( 66 | model=model, 67 | messages=[message], 68 | ) 69 | 70 | 71 | def benchmark_prompt_eval(model: str, df: pd.DataFrame) -> None: 72 | messages = MESSAGES["prompt_eval_messages"][:MESSAGE_LIMIT] 73 | for message in tqdm(messages): 74 | df.loc[len(df)] = ollama.chat( 75 | model=model, 76 | messages=[message], 77 | ) 78 | 79 | 80 | def benchmarks() -> None: 81 | # Eval Benchmark 82 | if not EVAL_PATH.exists(): 83 | eval_df = pd.DataFrame(columns=COLUMNS) 84 | for model in MODELS: 85 | benchmark_eval(model, eval_df) 86 | eval_df.to_csv(EVAL_PATH, index=False) 87 | 88 | # Prompt Eval Benchmark 89 | if not PROMPT_EVAL_PATH.exists(): 90 | prompt_eval_df = pd.DataFrame(columns=COLUMNS) 91 | for model in MODELS: 92 | benchmark_prompt_eval(model, prompt_eval_df) 93 | prompt_eval_df.to_csv(PROMPT_EVAL_PATH, index=False) 94 | 95 | 96 | def plot(df: pd.DataFrame, plot_path: Path) -> None: 97 | # Set the theme and color palette 98 | sns.set_theme(style="whitegrid") 99 | gray_color = "#787878" # A mid-tone gray 100 | 101 | # Create a bar plot 102 | fig, ax = plt.subplots(figsize=(8, len(MODELS) * 0.6)) 103 | sns.barplot( 104 | x="speed", 105 | y="model", 106 | data=df, 107 | ax=ax, 108 | hue="model", 109 | edgecolor=gray_color, 110 | ) 111 | 112 | # Set labels 113 | ax.set_xlabel("tokens/s", color=gray_color) 114 | ax.set_ylabel("", color=gray_color) 115 | 116 | # Set tick parameters 117 | ax.tick_params(axis="x", colors=gray_color) 118 | ax.tick_params(axis="y", colors=gray_color) 119 | 120 | # Set the background color of the plot to None (transparent) 121 | fig.patch.set_facecolor("none") 122 | ax.set_facecolor("none") 123 | 124 | # Remove axes' frames for aesthetics 125 | for spine in ax.spines.values(): 126 | spine.set_visible(False) 127 | 128 | # Customize grid lines to the same shade of gray 129 | ax.xaxis.grid(True, color=gray_color, linestyle="-", linewidth=0.75) 130 | ax.yaxis.grid(False) 131 | 132 | # Save the plot with a transparent background 133 | plt.savefig( 134 | plot_path, 135 | bbox_inches="tight", 136 | pad_inches=0, 137 | transparent=True, 138 | ) 139 | 140 | 141 | def plots() -> None: 142 | # Eval Plot 143 | df = pd.read_csv(EVAL_PATH) 144 | df["speed"] = df["eval_count"] / df["eval_duration"] * 1e9 145 | plot(df, BENCHMARKS_PATH / "plot_eval.svg") 146 | 147 | # Eval Plot 148 | df = pd.read_csv(PROMPT_EVAL_PATH) 149 | df["speed"] = df["prompt_eval_count"] / df["prompt_eval_duration"] * 1e9 150 | plot(df, BENCHMARKS_PATH / "plot_prompt_eval.svg") 151 | 152 | 153 | def main() -> None: 154 | benchmarks() 155 | plots() 156 | 157 | 158 | if __name__ == "__main__": 159 | main() 160 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | # My ollama notes 4 | 5 | ![Figure from Ollama blog post about embeddings](https://github.com/S1M0N38/my-ollama-notes/blob/main/ollama-taking-notes.svg?raw=true "Figure from ollama.com/blog/embedding-models") 6 | 7 |
8 | 9 | [Ollama](https://ollama.com/) is one of the simplest ways to run Large Language Models (LLMs) on your hardware. 10 | 11 | Follow the installation guide on the official website or, if you are on Linux, simply download the single binary and make it executable: 12 | 13 | ```bash 14 | curl -L https://ollama.com/download/ollama-linux-amd64 -o ollama 15 | chmod +x ollama 16 | ``` 17 | 18 | Then start the Ollama server (and keep it running in the background): 19 | 20 | ```bash 21 | ./ollama serve 22 | ``` 23 | 24 | Now you can interact with the Ollama server in various ways: 25 | 26 | - [Command line interface](https://github.com/ollama/ollama/blob/main/README.md#quickstart): In a new terminal, run ./ollama run --help 27 | - [REST API](https://github.com/ollama/ollama/blob/main/docs/api.md): Perform HTTP requests to `localhost:11434` 28 | - [ollama-python](https://github.com/ollama/ollama-python): Ollama's Python library API, a wrapper around the REST API 29 | - [Open WebUI](https://docs.openwebui.com/): A ChatGPT-like web interface for Ollama 30 | 31 | ## Notes 32 | 33 | In order to have fast inference, the models must fit into the GPU memory. (The underlying inference engine is [llama.cpp](https://github.com/ggerganov/llama.cpp), which is able to run entire/partial models on CPU as well, but it is orders of magnitude slower.) 34 | 35 | **Size**: The most limiting factor in the choice of model is the amount of VRAM available on the GPU. For practical purposes, the models can be categorized into three groups based on the number of parameters. *Small* models can be run on a single consumer GPU (possibly locally on a laptop). *Medium* models usually require dedicated hardware with a decent amount of VRAM (e.g. > 40 GB). *Large* models require high-end GPU/s (80 GB or more combined VRAM). We assume the models are quantized to 4-bit precision (see later). 36 | 37 | - *Small* (\< 13B) 38 | - [llama3:8b](https://ollama.com/library/llama3) 39 | - [gemma:7b](https://ollama.com/library/gemma) 40 | - [mistral:7b](https://ollama.com/library/mistral) 41 | - [wizardlm2:7b](https://ollama.com/library/wizardlm2) 42 | - ... 43 | - *Medium* (13B - 70B) 44 | - [llama3:70b](https://ollama.com/library/llama3) 45 | - [command-r:35b](https://ollama.com/library/command-r) 46 | - [mixtral:8x7b](https://ollama.com/library/mixtral) 47 | - ... 48 | - *Large* (>70B) 49 | - [command-r-plus:104b](https://ollama.com/library/command-r-plus) 50 | - [mixtral:8x22b](https://ollama.com/library/mixtral) 51 | - [wizardlm2:8x22b](https://ollama.com/library/wizardlm2) 52 | - ... 53 | 54 | **MoE vs Dense**: Some of the models express the number of parameters as a multiplication (e.g. `mixtral:8x7b`). These models are referred to as Mixture of Experts (MoE) and at inference time, a routing network is used to select a subset of the experts to run (e.g. 2 out of 8) for each token. This effectively reduces the number of parameters used in each forward pass, making token generation faster with minimal loss in performance compared to dense models. 55 | 56 | **Quantization**: To run reasonably large models on a single GPU, the models are quantized with various precisions and methods. When pulling models with a simple tag (e.g. `ollama pull mistral`), it defaults to pulling the model with 4-bit quantization. See the Ollama website for all the available tags for each model. 57 | 58 | **Model Format**: Under the hood, Ollama makes use of llama.cpp which requires its custom format for model weights and specifications: `.gguf`. The tags follow the `gguf` naming scheme. 59 | 60 | **License**: Various models are released under different licenses. For example, models from Mistral are released under the "Apache 2.0" license, a very permissive license. The models from the Command-R series are released under the "CC-BY-NC" license, which is more restrictive, limiting the use of the models for commercial purposes (unless you purchase a commercial license). 61 | 62 | **Capabilities**: Different models excel at different tasks: code generation, multi-language understanding, reasoning capabilities, RAG performance, tool usage, and context length, just to name a few. Some respond in a more casual and engaging manner, while others are more formal and informative. 63 | 64 | ## Benchmarks 65 | 66 | Benchmarking model performance and capabilities is quite challenging. In addition to the usual metrics on popular datasets (MMLU, GPQA, HumanEval, GSM-8K, etc.), the [chat bot arena](https://chat.lmsys.org/?leaderboard) provides an ELO-based ranking based on human evaluations of the generated text. Another resource for LLM public perception is the [LocalLLaMa](https://www.reddit.com/r/LocalLLaMA/) subreddit. 67 | 68 | Below are some speed benchmarks for several base models available on Ollama. The plots were produced by running `benchmarks.py` on a machine equipped with: 69 | 70 |
71 | 72 | APPLE M1 MAX 73 | 74 | - **CPU**: APPLE M1 MAX 75 | - **RAM**: 32 GB 76 | 77 | ### Eval Speed 78 | 79 | How many tokens per second can the model generate in an autoregressive setting? 80 | 81 | ![Plot eval speed](https://github.com/S1M0N38/my-ollama-notes/blob/main/benchmarks/M1MAX/plot_eval.svg?raw=true) 82 | 83 | ### Prompt Eval Speed 84 | 85 | How fast (in tokens/s) can the model process a given prompt? 86 | 87 | ![Plot prompt eval speed](https://github.com/S1M0N38/my-ollama-notes/blob/main/benchmarks/M1MAX/plot_prompt_eval.svg?raw=true) 88 | 89 |
90 | 91 |
92 | 93 | NVIDIA 1080 TI 94 | 95 | - **CPU**: Intel Xeon CPU E5-2620 v3 @ 2.40GH (24 cores) 96 | - **RAM**: 126 GB 97 | 98 | ### Eval Speed 99 | 100 | How many tokens per second can the model generate in an autoregressive setting? 101 | 102 | ![Plot eval speed](https://github.com/S1M0N38/my-ollama-notes/blob/main/benchmarks/1080TI/plot_eval.svg?raw=true) 103 | 104 | ### Prompt Eval Speed 105 | 106 | How fast (in tokens/s) can the model process a given prompt? 107 | 108 | ![Plot prompt eval speed](https://github.com/S1M0N38/my-ollama-notes/blob/main/benchmarks/1080TI/plot_prompt_eval.svg?raw=true) 109 | 110 |
111 | 112 |
113 | 114 | NVIDIA A6000 115 | 116 | - **CPU**: AMD EPYC-Rome (14 cores) 117 | - **GPU**: NVIDIA A6000 (48 GB) 118 | - **RAM**: 92 GB 119 | 120 | ### Eval Speed 121 | 122 | How many tokens per second can the model generate in an autoregressive setting? 123 | 124 | ![Plot eval speed](https://github.com/S1M0N38/my-ollama-notes/blob/main/benchmarks/A6000/plot_eval.svg?raw=true) 125 | 126 | ### Prompt Eval Speed 127 | 128 | How fast (in tokens/s) can the model process a given prompt? 129 | 130 | ![Plot prompt eval speed](https://github.com/S1M0N38/my-ollama-notes/blob/main/benchmarks/A6000/plot_prompt_eval.svg?raw=true) 131 | 132 |
133 | 134 | ## Model Choice 135 | 136 | Even though the model zoo can be overwhelming with new models released regularly, finding a good model for your use case can be broken down into a few steps: 137 | 138 | 1. **Technical Limitations**: It must run at the required speed (size, MoE vs Dense). 139 | 1. **License Limitations**: It can be used for the intended purpose (license). 140 | 1. **Capabilities**: It should be good at the intended task (capabilities). 141 | 142 | Once you have identified a model or models that meet your requirements, you can further "optimize" your choice by considering the following: 143 | 144 | - **Quantization**: Move to a less quantized model if you have spare VRAM (while keeping an eye on the speed). 145 | - **Fine-tuned version**: The LLM community often releases fine-tuned versions of the base models for specific tasks (e.g., uncensored models, improved prompt following capabilities, etc.). 146 | - **Inference engine**: Ollama is easy to start with, but there exist more performant inference engines than llama.cpp (exllamav2, vllm, etc.). See [here](https://www.reddit.com/r/LocalLLaMA/comments/1anb2fz/guide_to_choosing_quants_and_engines/) and [here](https://www.reddit.com/r/LocalLLaMA/comments/1c9mn1n/whats_the_fastest_local_inference_engine_right/). 147 | -------------------------------------------------------------------------------- /benchmarks/1080TI/benchmarks_eval.csv: -------------------------------------------------------------------------------- 1 | model,total_duration,load_duration,prompt_eval_count,prompt_eval_duration,eval_count,eval_duration 2 | phi3:3.8b-mini-instruct-4k-q4_K_M,6437293717,2347770993,13,59218000,313,4029205000 3 | phi3:3.8b-mini-instruct-4k-q4_K_M,7330052202,677731,17,58044000,551,7269166000 4 | phi3:3.8b-mini-instruct-4k-q4_K_M,5401367348,693171,11,56401000,414,5342824000 5 | phi3:3.8b-mini-instruct-4k-q4_K_M,7501587967,691674,15,56920000,565,7442552000 6 | phi3:3.8b-mini-instruct-4k-q4_K_M,18739658915,664641,16,57268000,1292,18680439000 7 | phi3:3.8b-mini-instruct-4k-q4_K_M,8154533962,580058,12,57364000,608,8095287000 8 | phi3:3.8b-mini-instruct-4k-q4_K_M,9024532492,462416,14,57643000,668,8965480000 9 | phi3:3.8b-mini-instruct-4k-q4_K_M,9229873472,800869,11,58434000,684,9168953000 10 | phi3:3.8b-mini-instruct-4k-q4_K_M,9205804737,718318,14,58970000,679,9144452000 11 | phi3:3.8b-mini-instruct-4k-q4_K_M,9117887384,618181,12,57647000,673,9058118000 12 | phi3:3.8b-mini-instruct-4k-q4_K_M,6127463869,839728,12,57721000,464,6067185000 13 | phi3:3.8b-mini-instruct-4k-q4_K_M,3774568995,714478,9,57585000,291,3714928000 14 | phi3:3.8b-mini-instruct-4k-q4_K_M,11518036931,574461,13,57191000,835,11458910000 15 | phi3:3.8b-mini-instruct-4k-q4_K_M,5734938839,658778,12,57496000,434,5675431000 16 | phi3:3.8b-mini-instruct-4k-q4_K_M,13711710242,631924,12,58842000,971,13651029000 17 | phi3:3.8b-mini-instruct-4k-q4_K_M,8315752631,596342,11,58673000,613,8255321000 18 | phi3:3.8b-mini-instruct-4k-q4_K_M,6089656615,588381,14,59484000,457,6028301000 19 | phi3:3.8b-mini-instruct-4k-q4_K_M,7636213127,586955,11,60259000,563,7574144000 20 | phi3:3.8b-mini-instruct-4k-q4_K_M,4108848349,604724,9,58308000,313,4048640000 21 | phi3:3.8b-mini-instruct-4k-q4_K_M,5671203123,552221,10,58035000,427,5611431000 22 | phi3:3.8b-mini-instruct-4k-q4_K_M,10255212219,577545,13,58856000,742,10194567000 23 | phi3:3.8b-mini-instruct-4k-q4_K_M,10008435562,600841,14,58672000,725,9947886000 24 | phi3:3.8b-mini-instruct-4k-q4_K_M,6070728336,605181,14,58646000,455,6010113000 25 | phi3:3.8b-mini-instruct-4k-q4_K_M,8806868938,576688,12,60695000,644,8744101000 26 | phi3:3.8b-mini-instruct-4k-q4_K_M,5692652415,556085,13,60087000,425,5630752000 27 | phi3:3.8b-mini-instruct-4k-q4_K_M,6481725259,555258,11,60068000,484,6419920000 28 | phi3:3.8b-mini-instruct-4k-q4_K_M,10190949089,584775,13,60236000,738,10128670000 29 | phi3:3.8b-mini-instruct-4k-q4_K_M,12372384788,579351,13,59856000,880,12310588000 30 | phi3:3.8b-mini-instruct-4k-q4_K_M,3135857314,565122,14,60789000,240,3073196000 31 | phi3:3.8b-mini-instruct-4k-q4_K_M,11563413624,631858,13,58031000,828,11503610000 32 | phi3:3.8b-mini-instruct-4k-q4_K_M,10405534627,565722,13,58963000,751,10344667000 33 | phi3:3.8b-mini-instruct-4k-q4_K_M,8003385011,594085,13,58967000,589,7942595000 34 | phi3:3.8b-mini-instruct-4k-q4_K_M,9148750335,557178,13,58764000,667,9088351000 35 | phi3:3.8b-mini-instruct-4k-q4_K_M,4974749421,836485,14,60348000,375,4911722000 36 | phi3:3.8b-mini-instruct-4k-q4_K_M,5771716007,559081,15,59792000,433,5710257000 37 | phi3:3.8b-mini-instruct-4k-q4_K_M,7852109927,647834,11,59641000,579,7790669000 38 | phi3:3.8b-mini-instruct-4k-q4_K_M,9871779590,559031,11,59407000,717,9810666000 39 | phi3:3.8b-mini-instruct-4k-q4_K_M,8624049498,564145,15,58958000,632,8563361000 40 | phi3:3.8b-mini-instruct-4k-q4_K_M,9669642167,862425,15,60612000,702,9606586000 41 | phi3:3.8b-mini-instruct-4k-q4_K_M,8039781014,563391,13,58293000,593,7979694000 42 | llama3:8b-instruct-q4_0,13252984837,5590331379,16,100447000,368,7561065000 43 | llama3:8b-instruct-q4_0,11543209435,580867,16,97800000,548,11443168000 44 | llama3:8b-instruct-q4_0,10062326056,577981,12,97469000,478,9962771000 45 | llama3:8b-instruct-q4_0,11347451371,943832,13,99082000,539,11244346000 46 | llama3:8b-instruct-q4_0,11779751040,866128,14,99510000,557,11676113000 47 | llama3:8b-instruct-q4_0,12561000269,797835,13,98135000,592,12459865000 48 | llama3:8b-instruct-q4_0,12013251612,949471,13,99191000,568,11910122000 49 | llama3:8b-instruct-q4_0,9643158684,765588,11,98908000,459,9541344000 50 | llama3:8b-instruct-q4_0,14807794924,791838,13,98177000,693,14706626000 51 | llama3:8b-instruct-q4_0,10685615121,884031,12,96875000,508,10585895000 52 | llama3:8b-instruct-q4_0,14894426886,585084,13,101305000,697,14790194000 53 | llama3:8b-instruct-q4_0,10067699983,633198,10,98944000,477,9965948000 54 | llama3:8b-instruct-q4_0,14755344262,823041,12,97542000,694,14653850000 55 | llama3:8b-instruct-q4_0,11607896422,784625,13,97221000,549,11507525000 56 | llama3:8b-instruct-q4_0,12388978348,2667950,13,99825000,582,12283771000 57 | llama3:8b-instruct-q4_0,11412095711,622318,12,97295000,539,11312204000 58 | llama3:8b-instruct-q4_0,12692065310,869639,14,97809000,598,12589831000 59 | llama3:8b-instruct-q4_0,10915116848,731868,12,97308000,519,10814934000 60 | llama3:8b-instruct-q4_0,7439614173,608174,10,97490000,356,7339677000 61 | llama3:8b-instruct-q4_0,10096137220,600158,10,97821000,480,9996058000 62 | llama3:8b-instruct-q4_0,11328974850,643418,13,97275000,537,11229057000 63 | llama3:8b-instruct-q4_0,11944574970,644475,14,97407000,564,11844545000 64 | llama3:8b-instruct-q4_0,9217364566,624737,13,97425000,440,9117221000 65 | llama3:8b-instruct-q4_0,11661116415,671674,13,97854000,552,11560392000 66 | llama3:8b-instruct-q4_0,9147891364,869288,14,97683000,435,9046194000 67 | llama3:8b-instruct-q4_0,11601749604,659155,10,97310000,549,11501959000 68 | llama3:8b-instruct-q4_0,10065868277,599184,12,97883000,478,9965333000 69 | llama3:8b-instruct-q4_0,14841913941,613028,13,97391000,695,14741681000 70 | llama3:8b-instruct-q4_0,4009376193,679225,15,97445000,193,3909166000 71 | llama3:8b-instruct-q4_0,10599439011,578571,13,97714000,502,10499163000 72 | llama3:8b-instruct-q4_0,11627808252,405337,14,100405000,548,11525447000 73 | llama3:8b-instruct-q4_0,9152929561,743848,14,99473000,436,9050581000 74 | llama3:8b-instruct-q4_0,12078708529,1801073,13,97678000,570,11976145000 75 | llama3:8b-instruct-q4_0,11466255491,727068,13,97412000,542,11365923000 76 | llama3:8b-instruct-q4_0,11513640865,620525,13,100050000,544,11411109000 77 | llama3:8b-instruct-q4_0,8024185433,657528,12,97879000,384,7923707000 78 | llama3:8b-instruct-q4_0,13255077854,1646558,12,101992000,625,13149853000 79 | llama3:8b-instruct-q4_0,9731119117,395608,15,97402000,462,9631264000 80 | llama3:8b-instruct-q4_0,15430188991,605987,14,99148000,721,15328174000 81 | llama3:8b-instruct-q4_0,9827951011,640735,13,97886000,467,9727630000 82 | gemma:7b-instruct-v1.1-q4_0,12980828302,5989163847,15,95299000,317,6895359000 83 | gemma:7b-instruct-v1.1-q4_0,10590221046,675913,16,93999000,475,10494105000 84 | gemma:7b-instruct-v1.1-q4_0,7809454030,870882,11,94239000,352,7712174000 85 | gemma:7b-instruct-v1.1-q4_0,9436552003,801431,13,91786000,424,9342175000 86 | gemma:7b-instruct-v1.1-q4_0,10585863441,776852,13,92578000,473,10490540000 87 | gemma:7b-instruct-v1.1-q4_0,9751685923,849378,12,92571000,437,9656093000 88 | gemma:7b-instruct-v1.1-q4_0,8119026127,782259,13,92120000,365,8024201000 89 | gemma:7b-instruct-v1.1-q4_0,8079411997,695608,10,91404000,363,7985528000 90 | gemma:7b-instruct-v1.1-q4_0,9049813990,777188,13,94027000,404,8953229000 91 | gemma:7b-instruct-v1.1-q4_0,7481911326,703808,12,91379000,337,7387757000 92 | gemma:7b-instruct-v1.1-q4_0,7672289090,711888,13,91060000,345,7578778000 93 | gemma:7b-instruct-v1.1-q4_0,8268204715,717281,10,90794000,372,8175023000 94 | gemma:7b-instruct-v1.1-q4_0,9090289721,683978,12,91285000,408,8996632000 95 | gemma:7b-instruct-v1.1-q4_0,9456795872,679436,13,91390000,423,9363139000 96 | gemma:7b-instruct-v1.1-q4_0,10443577186,663905,13,91083000,467,10350265000 97 | gemma:7b-instruct-v1.1-q4_0,8509021072,452718,11,92624000,382,8414778000 98 | gemma:7b-instruct-v1.1-q4_0,9461327608,813142,14,93393000,423,9364942000 99 | gemma:7b-instruct-v1.1-q4_0,8868412005,656775,12,95132000,397,8770864000 100 | gemma:7b-instruct-v1.1-q4_0,6994241470,623758,10,91484000,315,6900455000 101 | gemma:7b-instruct-v1.1-q4_0,8368876295,601125,10,92260000,375,8274347000 102 | gemma:7b-instruct-v1.1-q4_0,9698783252,798395,13,92201000,433,9603485000 103 | gemma:7b-instruct-v1.1-q4_0,9198030703,760439,14,93422000,412,9101947000 104 | gemma:7b-instruct-v1.1-q4_0,7604967085,607278,13,93088000,342,7509387000 105 | gemma:7b-instruct-v1.1-q4_0,8736998311,650511,12,91443000,392,8643286000 106 | gemma:7b-instruct-v1.1-q4_0,7755741577,667215,14,94623000,348,7659016000 107 | gemma:7b-instruct-v1.1-q4_0,8332383746,624649,10,91033000,374,8239428000 108 | gemma:7b-instruct-v1.1-q4_0,10569054178,652989,12,90926000,472,10475586000 109 | gemma:7b-instruct-v1.1-q4_0,9660334344,598605,13,92923000,432,9565349000 110 | gemma:7b-instruct-v1.1-q4_0,3523685997,588424,15,93657000,159,3427734000 111 | gemma:7b-instruct-v1.1-q4_0,10268844601,598445,13,93007000,459,10173749000 112 | gemma:7b-instruct-v1.1-q4_0,7644398103,882372,14,92426000,343,7549003000 113 | gemma:7b-instruct-v1.1-q4_0,4205091302,775022,14,91251000,190,4111273000 114 | gemma:7b-instruct-v1.1-q4_0,7680777761,654239,13,92371000,345,7586135000 115 | gemma:7b-instruct-v1.1-q4_0,7815343185,676318,12,94758000,351,7718245000 116 | gemma:7b-instruct-v1.1-q4_0,9277360938,2046508,14,92742000,416,9180211000 117 | gemma:7b-instruct-v1.1-q4_0,7711191438,620978,12,91556000,348,7617525000 118 | gemma:7b-instruct-v1.1-q4_0,9093448917,644012,12,91417000,408,8999975000 119 | gemma:7b-instruct-v1.1-q4_0,9342546002,620205,15,91903000,419,9248502000 120 | gemma:7b-instruct-v1.1-q4_0,9617038044,672868,13,92898000,431,9521958000 121 | gemma:7b-instruct-v1.1-q4_0,10536845915,606918,13,90993000,471,10443839000 122 | mistral:7b-instruct-v0.2-q4_0,4717171604,2542310849,15,92843000,126,2081055000 123 | mistral:7b-instruct-v0.2-q4_0,10315239306,590402,15,91431000,585,10222087000 124 | mistral:7b-instruct-v0.2-q4_0,5423686851,566678,11,91604000,314,5330109000 125 | mistral:7b-instruct-v0.2-q4_0,9181035481,887705,12,93429000,523,9084883000 126 | mistral:7b-instruct-v0.2-q4_0,13830949858,860719,15,91519000,771,13736687000 127 | mistral:7b-instruct-v0.2-q4_0,4559027838,837118,12,91498000,264,4464221000 128 | mistral:7b-instruct-v0.2-q4_0,9922384453,750822,13,91972000,563,9828125000 129 | mistral:7b-instruct-v0.2-q4_0,6647549658,831046,12,91841000,382,6553067000 130 | mistral:7b-instruct-v0.2-q4_0,7891692977,751129,12,93442000,451,7795825000 131 | mistral:7b-instruct-v0.2-q4_0,7383429684,812695,11,92053000,424,7288565000 132 | mistral:7b-instruct-v0.2-q4_0,9603906257,858789,12,91763000,546,9509408000 133 | mistral:7b-instruct-v0.2-q4_0,3434596764,823799,9,91039000,199,3340954000 134 | mistral:7b-instruct-v0.2-q4_0,9639145138,702638,12,92359000,548,9544488000 135 | mistral:7b-instruct-v0.2-q4_0,8822672501,764526,12,91568000,503,8728750000 136 | mistral:7b-instruct-v0.2-q4_0,7792418309,770089,12,93613000,446,7696335000 137 | mistral:7b-instruct-v0.2-q4_0,8226453702,788145,11,91985000,470,8132294000 138 | mistral:7b-instruct-v0.2-q4_0,7358877435,620141,13,91875000,423,7265004000 139 | mistral:7b-instruct-v0.2-q4_0,7080100585,612725,11,91362000,407,6986703000 140 | mistral:7b-instruct-v0.2-q4_0,2117063299,607311,9,91964000,122,2023272000 141 | mistral:7b-instruct-v0.2-q4_0,8382897754,654275,9,91195000,478,8289957000 142 | mistral:7b-instruct-v0.2-q4_0,12157866737,655861,13,91882000,683,12064013000 143 | mistral:7b-instruct-v0.2-q4_0,11839489414,596835,14,94399000,666,11743295000 144 | mistral:7b-instruct-v0.2-q4_0,10807629415,586008,12,91665000,611,10714213000 145 | mistral:7b-instruct-v0.2-q4_0,9256373624,561358,13,91462000,526,9163127000 146 | mistral:7b-instruct-v0.2-q4_0,4669461133,656265,13,92822000,270,4574241000 147 | mistral:7b-instruct-v0.2-q4_0,6448443806,560614,10,91186000,371,6355225000 148 | mistral:7b-instruct-v0.2-q4_0,8997938913,667768,14,91890000,512,8903971000 149 | mistral:7b-instruct-v0.2-q4_0,9881890299,665624,13,91472000,560,9788481000 150 | mistral:7b-instruct-v0.2-q4_0,3950856208,596042,14,92474000,229,3856701000 151 | mistral:7b-instruct-v0.2-q4_0,12586640690,334831,12,93393000,705,12491655000 152 | mistral:7b-instruct-v0.2-q4_0,8883760571,594345,13,91751000,506,8790272000 153 | mistral:7b-instruct-v0.2-q4_0,6486209491,633554,13,91582000,373,6392856000 154 | mistral:7b-instruct-v0.2-q4_0,8547747599,637531,13,93259000,488,8452688000 155 | mistral:7b-instruct-v0.2-q4_0,6964543740,2176124,13,92225000,400,6869248000 156 | mistral:7b-instruct-v0.2-q4_0,8184605042,600464,13,95233000,468,8087596000 157 | mistral:7b-instruct-v0.2-q4_0,4840140879,674998,11,93918000,280,4744442000 158 | mistral:7b-instruct-v0.2-q4_0,7090542158,617205,11,91075000,408,6997699000 159 | mistral:7b-instruct-v0.2-q4_0,9517497872,572964,15,97163000,541,9418412000 160 | mistral:7b-instruct-v0.2-q4_0,8072118578,712351,14,91810000,461,7978248000 161 | mistral:7b-instruct-v0.2-q4_0,8737653523,641572,12,92584000,498,8643176000 162 | wizardlm2:7b-q4_0,13612503481,2719149995,44,97240000,614,10794743000 163 | wizardlm2:7b-q4_0,17720999346,591939,16,92795000,967,17625793000 164 | wizardlm2:7b-q4_0,14717994399,604132,12,93667000,814,14621973000 165 | wizardlm2:7b-q4_0,18764713853,848759,13,93575000,1021,18667477000 166 | wizardlm2:7b-q4_0,20137484918,683928,16,92534000,1086,20042443000 167 | wizardlm2:7b-q4_0,10769329784,640408,13,93361000,605,10673536000 168 | wizardlm2:7b-q4_0,15930248833,636192,14,92784000,875,15835106000 169 | wizardlm2:7b-q4_0,14189013901,649495,13,92675000,785,14094061000 170 | wizardlm2:7b-q4_0,17409740651,635062,13,94113000,950,17313168000 171 | wizardlm2:7b-q4_0,18117416558,658808,12,92379000,986,18022661000 172 | wizardlm2:7b-q4_0,17228034477,600805,13,93351000,942,17132296000 173 | wizardlm2:7b-q4_0,11559175450,630025,10,92343000,647,11464480000 174 | wizardlm2:7b-q4_0,24235308214,640812,13,93446000,1288,24139462000 175 | wizardlm2:7b-q4_0,17727744718,600595,13,91989000,967,17633399000 176 | wizardlm2:7b-q4_0,24841117013,1213006,13,93790000,1315,24744001000 177 | wizardlm2:7b-q4_0,14445219502,648275,12,92472000,799,14350306000 178 | wizardlm2:7b-q4_0,17878739886,625718,14,92868000,974,17783439000 179 | wizardlm2:7b-q4_0,17427023485,720766,12,92534000,952,17332044000 180 | wizardlm2:7b-q4_0,11999836310,601941,10,92917000,671,11904573000 181 | wizardlm2:7b-q4_0,14309793077,639708,10,92735000,793,14214821000 182 | wizardlm2:7b-q4_0,16946386403,611238,14,92330000,927,16851875000 183 | wizardlm2:7b-q4_0,18483439660,1189982,15,95726000,1004,18384459000 184 | wizardlm2:7b-q4_0,16295966183,612751,13,92515000,894,16201081000 185 | wizardlm2:7b-q4_0,16333066555,585011,14,93209000,896,16237577000 186 | wizardlm2:7b-q4_0,11660412886,613035,14,93672000,653,11564437000 187 | wizardlm2:7b-q4_0,17095725213,594528,11,91956000,936,17001409000 188 | wizardlm2:7b-q4_0,14191938979,904036,15,94543000,785,14093133000 189 | wizardlm2:7b-q4_0,22947427541,861195,14,93078000,1225,22850466000 190 | wizardlm2:7b-q4_0,8777351276,699604,15,94546000,498,8679047000 191 | wizardlm2:7b-q4_0,18505779738,402394,13,92071000,1007,18412150000 192 | wizardlm2:7b-q4_0,17663266991,727179,14,93939000,965,17566804000 193 | wizardlm2:7b-q4_0,13016391384,665673,14,92936000,726,12920980000 194 | wizardlm2:7b-q4_0,19656066086,651082,14,93192000,1064,19560285000 195 | wizardlm2:7b-q4_0,12522904255,666929,14,92922000,699,12427648000 196 | wizardlm2:7b-q4_0,16480006298,635183,14,92320000,904,16385405000 197 | wizardlm2:7b-q4_0,17143342067,1351022,12,95337000,938,17044682000 198 | wizardlm2:7b-q4_0,22004567671,636146,12,92637000,1180,21909478000 199 | wizardlm2:7b-q4_0,17968329176,622056,16,94191000,979,17871719000 200 | wizardlm2:7b-q4_0,20804055620,1197441,15,93110000,1120,20708417000 201 | wizardlm2:7b-q4_0,19400407819,592699,13,92067000,1051,19306056000 202 | -------------------------------------------------------------------------------- /benchmarks/1080TI/benchmarks_prompt_eval.csv: -------------------------------------------------------------------------------- 1 | model,total_duration,load_duration,prompt_eval_count,prompt_eval_duration,eval_count,eval_duration 2 | phi3:3.8b-mini-instruct-4k-q4_K_M,7021213880,1650226024,153,145191000,388,5224114000 3 | phi3:3.8b-mini-instruct-4k-q4_K_M,1586665771,590661,397,320751000,92,1260618000 4 | phi3:3.8b-mini-instruct-4k-q4_K_M,3199642559,586628,298,245174000,216,2948973000 5 | phi3:3.8b-mini-instruct-4k-q4_K_M,17301161264,581448,468,370399000,1075,16925789000 6 | phi3:3.8b-mini-instruct-4k-q4_K_M,8678102568,852729,389,322481000,569,8347538000 7 | phi3:3.8b-mini-instruct-4k-q4_K_M,8538326783,759275,243,196120000,587,8337314000 8 | phi3:3.8b-mini-instruct-4k-q4_K_M,4323551172,3084274,274,239347000,296,4076440000 9 | phi3:3.8b-mini-instruct-4k-q4_K_M,2616231943,598148,298,244408000,174,2367275000 10 | phi3:3.8b-mini-instruct-4k-q4_K_M,11337133579,674695,531,466750000,705,10863167000 11 | phi3:3.8b-mini-instruct-4k-q4_K_M,6157032952,1085872,286,244913000,419,5903482000 12 | phi3:3.8b-mini-instruct-4k-q4_K_M,7553906056,711725,300,246264000,511,7301632000 13 | phi3:3.8b-mini-instruct-4k-q4_K_M,6486190557,648002,278,240106000,443,6240573000 14 | phi3:3.8b-mini-instruct-4k-q4_K_M,8182717892,657835,311,251276000,548,7926267000 15 | phi3:3.8b-mini-instruct-4k-q4_K_M,10928538341,672214,524,465185000,682,10455731000 16 | phi3:3.8b-mini-instruct-4k-q4_K_M,10255871257,616204,521,465649000,641,9783649000 17 | phi3:3.8b-mini-instruct-4k-q4_K_M,2217490531,1816208,440,329364000,134,1880510000 18 | phi3:3.8b-mini-instruct-4k-q4_K_M,5299698759,597224,532,465890000,327,4826251000 19 | phi3:3.8b-mini-instruct-4k-q4_K_M,9671067115,646148,505,377522000,612,9286588000 20 | phi3:3.8b-mini-instruct-4k-q4_K_M,13193832582,1477430,492,374565000,828,12813788000 21 | phi3:3.8b-mini-instruct-4k-q4_K_M,9858527271,919749,495,379981000,625,9469279000 22 | phi3:3.8b-mini-instruct-4k-q4_K_M,9955279200,767051,532,464038000,622,9483294000 23 | phi3:3.8b-mini-instruct-4k-q4_K_M,12548949634,2463805,336,283950000,820,12256184000 24 | phi3:3.8b-mini-instruct-4k-q4_K_M,10063826425,631174,456,369900000,644,9686780000 25 | phi3:3.8b-mini-instruct-4k-q4_K_M,9883532092,659678,545,470962000,615,9404441000 26 | phi3:3.8b-mini-instruct-4k-q4_K_M,9408961655,635032,526,462831000,588,8938769000 27 | phi3:3.8b-mini-instruct-4k-q4_K_M,7019428299,615818,255,201999000,484,6812549000 28 | phi3:3.8b-mini-instruct-4k-q4_K_M,8347158478,611224,255,202880000,573,8139661000 29 | phi3:3.8b-mini-instruct-4k-q4_K_M,2473530856,601441,268,240739000,165,2227962000 30 | phi3:3.8b-mini-instruct-4k-q4_K_M,4718131827,722741,274,240168000,324,4473440000 31 | phi3:3.8b-mini-instruct-4k-q4_K_M,8247187619,797755,251,196316000,566,8043872000 32 | phi3:3.8b-mini-instruct-4k-q4_K_M,4260984914,712578,304,245755000,289,4010511000 33 | phi3:3.8b-mini-instruct-4k-q4_K_M,4414799918,652898,227,193681000,310,4216994000 34 | phi3:3.8b-mini-instruct-4k-q4_K_M,8253787284,669668,277,240032000,562,8008780000 35 | phi3:3.8b-mini-instruct-4k-q4_K_M,2881068140,688188,168,147239000,207,2729512000 36 | phi3:3.8b-mini-instruct-4k-q4_K_M,9243371409,392083,216,192715000,638,9047704000 37 | phi3:3.8b-mini-instruct-4k-q4_K_M,4678747596,638188,214,190789000,330,4484246000 38 | phi3:3.8b-mini-instruct-4k-q4_K_M,13335224198,598868,242,194951000,894,13135353000 39 | phi3:3.8b-mini-instruct-4k-q4_K_M,11049392347,606625,155,144625000,769,10900855000 40 | phi3:3.8b-mini-instruct-4k-q4_K_M,7831902665,614178,177,148793000,552,7679061000 41 | phi3:3.8b-mini-instruct-4k-q4_K_M,14135360348,577778,187,150352000,955,13981264000 42 | llama3:8b-instruct-q4_0,10905024652,5299182639,143,235015000,258,5368250000 43 | llama3:8b-instruct-q4_0,5881918077,632889,327,461233000,252,5412145000 44 | llama3:8b-instruct-q4_0,6143354848,645971,269,383119000,270,5750956000 45 | llama3:8b-instruct-q4_0,11263013244,1925388,370,464633000,490,10784696000 46 | llama3:8b-instruct-q4_0,8397067129,892785,335,458643000,363,7924925000 47 | llama3:8b-instruct-q4_0,6563979094,942479,222,314835000,293,6236327000 48 | llama3:8b-instruct-q4_0,5461889796,1666547,239,313792000,243,5135863000 49 | llama3:8b-instruct-q4_0,7752375202,693951,243,320648000,347,7422600000 50 | llama3:8b-instruct-q4_0,11327110398,771429,454,624506000,478,10688496000 51 | llama3:8b-instruct-q4_0,7364221052,639478,238,316157000,330,7038618000 52 | llama3:8b-instruct-q4_0,7902740488,897385,260,385201000,350,7499512000 53 | llama3:8b-instruct-q4_0,4663637497,775896,256,324455000,204,4328035000 54 | llama3:8b-instruct-q4_0,7820341622,795322,261,383330000,346,7426041000 55 | llama3:8b-instruct-q4_0,10490315771,771515,432,543277000,447,9933058000 56 | llama3:8b-instruct-q4_0,8688985460,714931,418,536831000,369,8140506000 57 | llama3:8b-instruct-q4_0,9789762815,704681,366,466119000,424,9310490000 58 | llama3:8b-instruct-q4_0,7427700057,676572,433,540935000,313,6874676000 59 | llama3:8b-instruct-q4_0,7583538194,4965701,426,546037000,320,7020997000 60 | llama3:8b-instruct-q4_0,7615056361,1075406,411,534515000,322,7062490000 61 | llama3:8b-instruct-q4_0,8585328380,3082450,404,539032000,366,8031833000 62 | llama3:8b-instruct-q4_0,6612872158,1901295,423,540393000,277,6057911000 63 | llama3:8b-instruct-q4_0,5897612845,696561,283,388881000,257,5496428000 64 | llama3:8b-instruct-q4_0,6931454705,801925,378,469297000,296,6441587000 65 | llama3:8b-instruct-q4_0,10498744171,871662,463,627253000,442,9853712000 66 | llama3:8b-instruct-q4_0,7962787907,2208985,441,542051000,336,7404109000 67 | llama3:8b-instruct-q4_0,6107954588,617888,214,310369000,274,5789265000 68 | llama3:8b-instruct-q4_0,4383401797,1326807,197,308302000,194,4066747000 69 | llama3:8b-instruct-q4_0,3201916834,669658,238,316966000,137,2876340000 70 | llama3:8b-instruct-q4_0,3528835066,1388449,254,321048000,153,3200084000 71 | llama3:8b-instruct-q4_0,10833923520,665399,211,311306000,488,10514878000 72 | llama3:8b-instruct-q4_0,6389637174,851232,246,320481000,284,6055058000 73 | llama3:8b-instruct-q4_0,5107461687,716902,197,311555000,228,4785751000 74 | llama3:8b-instruct-q4_0,6370767451,683211,217,311690000,285,6047382000 75 | llama3:8b-instruct-q4_0,7537139915,613060,161,239361000,345,7291116000 76 | llama3:8b-instruct-q4_0,12345575030,3194618,179,242812000,560,12091629000 77 | llama3:8b-instruct-q4_0,6744961469,653268,186,240907000,307,6497150000 78 | llama3:8b-instruct-q4_0,8600881831,795566,223,314603000,387,8276453000 79 | llama3:8b-instruct-q4_0,4503997500,1077479,128,174911000,208,4322014000 80 | llama3:8b-instruct-q4_0,7216653008,591481,140,234675000,331,6975241000 81 | llama3:8b-instruct-q4_0,5093318972,571145,153,236689000,232,4849880000 82 | gemma:7b-instruct-v1.1-q4_0,10520622150,5854036321,142,261154000,200,4403489000 83 | gemma:7b-instruct-v1.1-q4_0,7542644903,1799104,310,438973000,311,7096206000 84 | gemma:7b-instruct-v1.1-q4_0,7796433446,789826,262,430686000,324,7359822000 85 | gemma:7b-instruct-v1.1-q4_0,9508054634,840392,379,530730000,386,8964853000 86 | gemma:7b-instruct-v1.1-q4_0,6758851916,6855343,323,516887000,271,6224887000 87 | gemma:7b-instruct-v1.1-q4_0,7511871854,721615,217,346902000,317,7157513000 88 | gemma:7b-instruct-v1.1-q4_0,7362367750,680318,248,352984000,309,7002786000 89 | gemma:7b-instruct-v1.1-q4_0,6431941169,629608,232,349221000,269,6076494000 90 | gemma:7b-instruct-v1.1-q4_0,6999237289,731645,445,602846000,274,6386836000 91 | gemma:7b-instruct-v1.1-q4_0,5706150033,883966,236,351259000,237,5348153000 92 | gemma:7b-instruct-v1.1-q4_0,6008590652,806595,262,429955000,246,5565584000 93 | gemma:7b-instruct-v1.1-q4_0,5698137268,872098,248,353167000,234,5336250000 94 | gemma:7b-instruct-v1.1-q4_0,9760923087,711375,267,430212000,407,9323351000 95 | gemma:7b-instruct-v1.1-q4_0,9175793256,2216826,416,598383000,366,8565759000 96 | gemma:7b-instruct-v1.1-q4_0,7627720331,730578,444,604901000,298,7014155000 97 | gemma:7b-instruct-v1.1-q4_0,8198332947,767645,361,526719000,331,7663483000 98 | gemma:7b-instruct-v1.1-q4_0,7456598696,728852,439,608438000,293,6839614000 99 | gemma:7b-instruct-v1.1-q4_0,9279370155,685301,418,602851000,368,8668256000 100 | gemma:7b-instruct-v1.1-q4_0,6574898968,906055,408,601847000,257,5956024000 101 | gemma:7b-instruct-v1.1-q4_0,5078663788,777602,390,591895000,195,4477420000 102 | gemma:7b-instruct-v1.1-q4_0,5976447799,2087652,410,596865000,232,5366837000 103 | gemma:7b-instruct-v1.1-q4_0,7866618128,447343,280,438057000,325,7423169000 104 | gemma:7b-instruct-v1.1-q4_0,8300766035,2107348,369,528904000,335,7762501000 105 | gemma:7b-instruct-v1.1-q4_0,8412635010,832402,447,607445000,330,7790770000 106 | gemma:7b-instruct-v1.1-q4_0,8529494477,769671,430,601769000,338,7917880000 107 | gemma:7b-instruct-v1.1-q4_0,7914943436,697161,204,376151000,332,7532684000 108 | gemma:7b-instruct-v1.1-q4_0,6270639809,796615,190,271330000,267,5993166000 109 | gemma:7b-instruct-v1.1-q4_0,6004040614,624715,248,353353000,250,5644762000 110 | gemma:7b-instruct-v1.1-q4_0,7282103858,464188,261,431880000,301,6846850000 111 | gemma:7b-instruct-v1.1-q4_0,8352661005,721411,202,343496000,352,8003636000 112 | gemma:7b-instruct-v1.1-q4_0,4967590577,608781,238,351336000,205,4611022000 113 | gemma:7b-instruct-v1.1-q4_0,6057737902,626371,189,268521000,258,5784410000 114 | gemma:7b-instruct-v1.1-q4_0,6933375308,639518,212,347007000,291,6580482000 115 | gemma:7b-instruct-v1.1-q4_0,7635875538,626322,160,262750000,328,7368744000 116 | gemma:7b-instruct-v1.1-q4_0,8284118026,648955,181,270531000,355,8008601000 117 | gemma:7b-instruct-v1.1-q4_0,5235560413,694465,169,263762000,222,4967218000 118 | gemma:7b-instruct-v1.1-q4_0,7772459657,613918,225,350037000,326,7417036000 119 | gemma:7b-instruct-v1.1-q4_0,8751626760,617285,125,187431000,381,8559716000 120 | gemma:7b-instruct-v1.1-q4_0,7772514486,654468,144,259631000,334,7508402000 121 | gemma:7b-instruct-v1.1-q4_0,5874799907,606145,151,261792000,251,5608706000 122 | mistral:7b-instruct-v0.2-q4_0,5358093802,3158353716,150,222475000,117,1975486000 123 | mistral:7b-instruct-v0.2-q4_0,7483256211,584945,368,437981000,385,7039974000 124 | mistral:7b-instruct-v0.2-q4_0,7778156450,595722,294,370273000,410,7402582000 125 | mistral:7b-instruct-v0.2-q4_0,5454059522,1662091,429,510004000,270,4938048000 126 | mistral:7b-instruct-v0.2-q4_0,8204801145,745171,370,439985000,422,7758459000 127 | mistral:7b-instruct-v0.2-q4_0,4489013665,773482,234,298864000,238,4183676000 128 | mistral:7b-instruct-v0.2-q4_0,7249318453,705369,267,361861000,383,6882560000 129 | mistral:7b-instruct-v0.2-q4_0,6016256519,385449,288,398469000,312,5613394000 130 | mistral:7b-instruct-v0.2-q4_0,5410746533,764135,488,595724000,260,4808364000 131 | mistral:7b-instruct-v0.2-q4_0,8255304223,2291356,254,306549000,441,7941635000 132 | mistral:7b-instruct-v0.2-q4_0,6414844142,1795311,273,365185000,337,6042852000 133 | mistral:7b-instruct-v0.2-q4_0,4838561496,711241,265,365772000,252,4467176000 134 | mistral:7b-instruct-v0.2-q4_0,8914663984,732645,283,365008000,470,8544139000 135 | mistral:7b-instruct-v0.2-q4_0,2086781099,894816,479,595998000,82,1480888000 136 | mistral:7b-instruct-v0.2-q4_0,8570258911,744415,488,597616000,424,7965750000 137 | mistral:7b-instruct-v0.2-q4_0,11423289468,732198,412,505451000,580,10910885000 138 | mistral:7b-instruct-v0.2-q4_0,2671162247,808519,483,601934000,113,2061269000 139 | mistral:7b-instruct-v0.2-q4_0,2908754308,685738,473,661720000,122,2239910000 140 | mistral:7b-instruct-v0.2-q4_0,6658801966,684085,447,515696000,332,6136339000 141 | mistral:7b-instruct-v0.2-q4_0,7010302659,676734,472,594106000,345,6409776000 142 | mistral:7b-instruct-v0.2-q4_0,11594720903,693522,489,604064000,576,10983532000 143 | mistral:7b-instruct-v0.2-q4_0,2759293579,713555,308,372987000,135,2380038000 144 | mistral:7b-instruct-v0.2-q4_0,16716827868,720895,424,555584000,838,16153964000 145 | mistral:7b-instruct-v0.2-q4_0,8357309590,1873315,521,764330000,401,7583416000 146 | mistral:7b-instruct-v0.2-q4_0,9878079707,664195,497,595724000,490,9274683000 147 | mistral:7b-instruct-v0.2-q4_0,2939148660,615032,234,298064000,151,2636431000 148 | mistral:7b-instruct-v0.2-q4_0,3172714534,668408,235,298974000,164,2869192000 149 | mistral:7b-instruct-v0.2-q4_0,5102892580,1750971,263,363694000,266,4733227000 150 | mistral:7b-instruct-v0.2-q4_0,2033815064,668179,275,361319000,96,1668002000 151 | mistral:7b-instruct-v0.2-q4_0,9288240355,564298,234,299919000,497,8983949000 152 | mistral:7b-instruct-v0.2-q4_0,8446376653,595985,267,363018000,447,8078978000 153 | mistral:7b-instruct-v0.2-q4_0,1967666185,639352,208,295278000,97,1667348000 154 | mistral:7b-instruct-v0.2-q4_0,7191868424,1666760,249,299909000,384,6885677000 155 | mistral:7b-instruct-v0.2-q4_0,5289003781,1332849,165,223888000,290,5060464000 156 | mistral:7b-instruct-v0.2-q4_0,5382739721,560128,195,294587000,289,5084065000 157 | mistral:7b-instruct-v0.2-q4_0,2556014131,597078,204,292297000,131,2260161000 158 | mistral:7b-instruct-v0.2-q4_0,4567800704,1654507,238,299567000,242,4262287000 159 | mistral:7b-instruct-v0.2-q4_0,10405448793,587588,141,222582000,569,10179256000 160 | mistral:7b-instruct-v0.2-q4_0,14423966220,614549,157,223556000,776,14196489000 161 | mistral:7b-instruct-v0.2-q4_0,3887937954,583649,167,226011000,211,3658091000 162 | wizardlm2:7b-q4_0,12213395289,2591860891,179,228099000,525,9391480000 163 | wizardlm2:7b-q4_0,11284485299,656268,369,444775000,578,10834005000 164 | wizardlm2:7b-q4_0,15210636492,640372,295,371722000,787,14833264000 165 | wizardlm2:7b-q4_0,12407279439,641062,430,510825000,625,11891036000 166 | wizardlm2:7b-q4_0,12826337815,645025,371,442788000,654,12376947000 167 | wizardlm2:7b-q4_0,13814645059,638552,235,302879000,727,13506621000 168 | wizardlm2:7b-q4_0,16175637464,597791,268,368266000,838,15802396000 169 | wizardlm2:7b-q4_0,20109267749,630885,289,373176000,1025,19730232000 170 | wizardlm2:7b-q4_0,14253439200,646694,489,614710000,703,13631220000 171 | wizardlm2:7b-q4_0,13660369904,648804,255,304636000,716,13349745000 172 | wizardlm2:7b-q4_0,16784790748,649142,274,367076000,867,16411276000 173 | wizardlm2:7b-q4_0,13855056855,888462,266,364960000,722,13482494000 174 | wizardlm2:7b-q4_0,18271117684,803989,284,367302000,937,17897535000 175 | wizardlm2:7b-q4_0,14132045243,746831,480,607282000,699,13516432000 176 | wizardlm2:7b-q4_0,17137657776,455566,489,614531000,842,16518090000 177 | wizardlm2:7b-q4_0,14672313039,648085,413,510464000,737,14154585000 178 | wizardlm2:7b-q4_0,15275715618,1229672,484,609975000,754,14659213000 179 | wizardlm2:7b-q4_0,17652848950,735052,474,599024000,869,17045945000 180 | wizardlm2:7b-q4_0,18386634992,649718,448,510973000,912,17868078000 181 | wizardlm2:7b-q4_0,13797009951,1947228,473,592733000,684,13195723000 182 | wizardlm2:7b-q4_0,17342643257,734558,490,612108000,852,16722927000 183 | wizardlm2:7b-q4_0,15561052561,1515627,309,374480000,800,15178969000 184 | wizardlm2:7b-q4_0,13325078853,694498,425,515949000,670,12801523000 185 | wizardlm2:7b-q4_0,14366881614,704455,522,723605000,700,13634528000 186 | wizardlm2:7b-q4_0,17579430665,653558,498,613496000,861,16957777000 187 | wizardlm2:7b-q4_0,16189197279,649351,235,303649000,845,15880361000 188 | wizardlm2:7b-q4_0,14553608015,652818,236,302525000,765,14245868000 189 | wizardlm2:7b-q4_0,9615213586,625654,264,364937000,506,9245099000 190 | wizardlm2:7b-q4_0,9185910677,619841,276,368013000,483,8813276000 191 | wizardlm2:7b-q4_0,13986294962,717399,235,302174000,736,13679094000 192 | wizardlm2:7b-q4_0,18075399707,620051,268,365798000,931,17704764000 193 | wizardlm2:7b-q4_0,10294688673,618811,209,295464000,549,9994236000 194 | wizardlm2:7b-q4_0,12447691550,622138,250,305625000,656,12136710000 195 | wizardlm2:7b-q4_0,9954809116,1369390,166,230547000,539,9718533000 196 | wizardlm2:7b-q4_0,15232403688,607752,196,290338000,804,14937238000 197 | wizardlm2:7b-q4_0,12161552255,617758,205,292243000,648,11865147000 198 | wizardlm2:7b-q4_0,20109437657,620978,239,304456000,1038,19799685000 199 | wizardlm2:7b-q4_0,17479063495,621838,142,221849000,928,17252869000 200 | wizardlm2:7b-q4_0,18934991464,620985,158,226548000,996,18704150000 201 | wizardlm2:7b-q4_0,14516519413,616702,168,229425000,775,14282756000 202 | -------------------------------------------------------------------------------- /benchmarks/M1MAX/benchmarks_eval.csv: -------------------------------------------------------------------------------- 1 | model,total_duration,load_duration,prompt_eval_count,prompt_eval_duration,eval_count,eval_duration 2 | phi3:3.8b-mini-instruct-4k-q4_K_M,5819767791,1900753166,13,93235000,231,3823417000 3 | phi3:3.8b-mini-instruct-4k-q4_K_M,12647041333,1609208,17,86528000,721,12557708000 4 | phi3:3.8b-mini-instruct-4k-q4_K_M,4861966584,1493375,11,84268000,286,4775010000 5 | phi3:3.8b-mini-instruct-4k-q4_K_M,9371239042,1432792,15,85288000,545,9283438000 6 | phi3:3.8b-mini-instruct-4k-q4_K_M,17750823542,1504667,16,84623000,991,17663066000 7 | phi3:3.8b-mini-instruct-4k-q4_K_M,6264147667,1003709,12,84150000,371,6177686000 8 | phi3:3.8b-mini-instruct-4k-q4_K_M,11123382750,1525917,14,85370000,641,11035007000 9 | phi3:3.8b-mini-instruct-4k-q4_K_M,13032041292,1529875,11,83556000,749,12945555000 10 | phi3:3.8b-mini-instruct-4k-q4_K_M,15322085375,1605000,14,84177000,851,15234871000 11 | phi3:3.8b-mini-instruct-4k-q4_K_M,7521515917,792333,12,83656000,437,7435952000 12 | phi3:3.8b-mini-instruct-4k-q4_K_M,11748237375,902958,12,82889000,666,11663167000 13 | phi3:3.8b-mini-instruct-4k-q4_K_M,3655964167,1169833,9,84389000,215,3569101000 14 | phi3:3.8b-mini-instruct-4k-q4_K_M,12071956667,768500,13,83287000,681,11986747000 15 | phi3:3.8b-mini-instruct-4k-q4_K_M,8727251209,858417,12,83932000,502,8641349000 16 | phi3:3.8b-mini-instruct-4k-q4_K_M,11582562542,1428167,12,83797000,659,11495941000 17 | phi3:3.8b-mini-instruct-4k-q4_K_M,10205247667,879750,11,83513000,586,10119643000 18 | phi3:3.8b-mini-instruct-4k-q4_K_M,9095061458,909917,14,84410000,522,9008602000 19 | phi3:3.8b-mini-instruct-4k-q4_K_M,8210627875,1507291,11,83732000,481,8123924000 20 | phi3:3.8b-mini-instruct-4k-q4_K_M,2907966792,1504792,9,82709000,173,2822232000 21 | phi3:3.8b-mini-instruct-4k-q4_K_M,10519414542,1532459,10,84248000,606,10432279000 22 | phi3:3.8b-mini-instruct-4k-q4_K_M,12942028250,783208,13,84458000,672,12855637000 23 | phi3:3.8b-mini-instruct-4k-q4_K_M,20610182041,2132875,14,121967000,752,20484569000 24 | phi3:3.8b-mini-instruct-4k-q4_K_M,17137946708,1661250,14,121844000,615,17012800000 25 | phi3:3.8b-mini-instruct-4k-q4_K_M,17677853500,1559917,12,122235000,657,17552400000 26 | phi3:3.8b-mini-instruct-4k-q4_K_M,12761304334,1603167,13,121323000,484,12636996000 27 | phi3:3.8b-mini-instruct-4k-q4_K_M,17530241166,1631625,11,120474000,607,17406795000 28 | phi3:3.8b-mini-instruct-4k-q4_K_M,17901499459,1039209,13,157225000,647,17741860000 29 | phi3:3.8b-mini-instruct-4k-q4_K_M,15080418250,1762958,13,120596000,566,14955862000 30 | phi3:3.8b-mini-instruct-4k-q4_K_M,6562717292,982917,14,127625000,253,6432877000 31 | phi3:3.8b-mini-instruct-4k-q4_K_M,17509833833,1626167,13,121917000,649,17384810000 32 | phi3:3.8b-mini-instruct-4k-q4_K_M,15778636250,1641167,13,123463000,593,15651839000 33 | phi3:3.8b-mini-instruct-4k-q4_K_M,9741916667,1594834,13,121150000,374,9617548000 34 | phi3:3.8b-mini-instruct-4k-q4_K_M,18170174875,1789375,13,120657000,680,18046128000 35 | phi3:3.8b-mini-instruct-4k-q4_K_M,9789662459,1528834,14,121807000,372,9664764000 36 | phi3:3.8b-mini-instruct-4k-q4_K_M,16414863500,1559000,15,123135000,614,16288501000 37 | phi3:3.8b-mini-instruct-4k-q4_K_M,18604398750,1582500,11,119922000,657,18481396000 38 | phi3:3.8b-mini-instruct-4k-q4_K_M,20033214375,1526875,11,119179000,728,19910999000 39 | phi3:3.8b-mini-instruct-4k-q4_K_M,14524214042,1584250,15,122754000,547,14398350000 40 | phi3:3.8b-mini-instruct-4k-q4_K_M,16009166875,1449709,15,123091000,596,15883136000 41 | phi3:3.8b-mini-instruct-4k-q4_K_M,13037252334,1588792,13,120288000,485,12913790000 42 | llama3:8b-instruct-q4_0,19237567000,6700523125,16,149040000,301,12386111000 43 | llama3:8b-instruct-q4_0,20643949291,1734708,16,184889000,508,20455139000 44 | llama3:8b-instruct-q4_0,19268087584,1723709,12,181134000,475,19083350000 45 | llama3:8b-instruct-q4_0,25334185000,1673208,13,182747000,620,25147910000 46 | llama3:8b-instruct-q4_0,27387800833,1561208,14,185887000,654,27198686000 47 | llama3:8b-instruct-q4_0,20068738542,1017417,13,168462000,493,19897912000 48 | llama3:8b-instruct-q4_0,25246563416,1656500,13,187286000,614,25055778000 49 | llama3:8b-instruct-q4_0,16964350542,1586917,11,183109000,416,16777940000 50 | llama3:8b-instruct-q4_0,26775111833,1401208,13,185515000,648,26617451000 51 | llama3:8b-instruct-q4_0,18143532833,1634708,12,184036000,441,17960800000 52 | llama3:8b-instruct-q4_0,27756486625,1568833,13,184340000,672,27570890000 53 | llama3:8b-instruct-q4_0,18075079917,1582208,10,182204000,442,17890038000 54 | llama3:8b-instruct-q4_0,55375779833,1585625,12,183440000,1303,55189866000 55 | llama3:8b-instruct-q4_0,23237159375,1544542,13,181868000,567,23051993000 56 | llama3:8b-instruct-q4_0,26842454333,1564292,13,182674000,652,26656766000 57 | llama3:8b-instruct-q4_0,25234933875,1623208,12,179664000,615,25052090000 58 | llama3:8b-instruct-q4_0,21167148500,1625958,14,184572000,519,20979303000 59 | llama3:8b-instruct-q4_0,19363170667,1738209,12,179078000,476,19180673000 60 | llama3:8b-instruct-q4_0,19107682167,1569459,10,181074000,470,18923504000 61 | llama3:8b-instruct-q4_0,19970416583,924500,10,183634000,492,19784722000 62 | llama3:8b-instruct-q4_0,19991813917,1524083,13,180799000,491,19807683000 63 | llama3:8b-instruct-q4_0,26307446333,1681208,14,183473000,644,26120556000 64 | llama3:8b-instruct-q4_0,15132512875,1600333,13,183822000,375,14945268000 65 | llama3:8b-instruct-q4_0,21893321500,1415875,13,182676000,539,21707646000 66 | llama3:8b-instruct-q4_0,17215638875,1929292,14,182474000,426,17029679000 67 | llama3:8b-instruct-q4_0,21919186416,1546333,10,179796000,540,21736478000 68 | llama3:8b-instruct-q4_0,16429048208,1640541,12,180851000,407,16244857000 69 | llama3:8b-instruct-q4_0,30051843416,1618208,13,182202000,733,29866613000 70 | llama3:8b-instruct-q4_0,6682050875,1547750,15,181073000,166,6497641000 71 | llama3:8b-instruct-q4_0,19810847584,1544334,13,184568000,489,19623216000 72 | llama3:8b-instruct-q4_0,21872383625,1614625,14,183184000,538,21686053000 73 | llama3:8b-instruct-q4_0,19559084542,1420958,14,183446000,482,19372654000 74 | llama3:8b-instruct-q4_0,20992503042,3622834,13,181930000,517,20805421000 75 | llama3:8b-instruct-q4_0,16704208250,1632333,13,182685000,413,16518250000 76 | llama3:8b-instruct-q4_0,23759952041,1597250,13,183906000,582,23572868000 77 | llama3:8b-instruct-q4_0,14470234958,1643500,12,179018000,358,14287989000 78 | llama3:8b-instruct-q4_0,24336094875,1650250,12,183131000,597,24149826000 79 | llama3:8b-instruct-q4_0,19819207792,1665125,15,179631000,488,19636280000 80 | llama3:8b-instruct-q4_0,26617338542,1593375,14,183669000,651,26430540000 81 | llama3:8b-instruct-q4_0,23397985792,1598334,13,181361000,573,23213560000 82 | gemma:7b-instruct-v1.1-q4_0,20819486709,7284173417,15,170083000,289,13363423000 83 | gemma:7b-instruct-v1.1-q4_0,21988153667,1842917,16,195357000,469,21788806000 84 | gemma:7b-instruct-v1.1-q4_0,20637387166,1615750,11,195532000,440,20438376000 85 | gemma:7b-instruct-v1.1-q4_0,18413031584,1724625,13,199836000,392,18209660000 86 | gemma:7b-instruct-v1.1-q4_0,21908994000,1683208,13,200789000,457,21704706000 87 | gemma:7b-instruct-v1.1-q4_0,21423547375,1580750,12,326177000,428,21093943000 88 | gemma:7b-instruct-v1.1-q4_0,16167427666,1524250,13,190219000,345,15974204000 89 | gemma:7b-instruct-v1.1-q4_0,17996537625,1735375,10,197767000,384,17794668000 90 | gemma:7b-instruct-v1.1-q4_0,20698254708,1555166,13,199078000,441,20494071000 91 | gemma:7b-instruct-v1.1-q4_0,16927072125,1517500,12,197480000,362,16726447000 92 | gemma:7b-instruct-v1.1-q4_0,16027516917,1913417,13,201051000,342,15823069000 93 | gemma:7b-instruct-v1.1-q4_0,21264070375,1598792,10,197694000,453,21063146000 94 | gemma:7b-instruct-v1.1-q4_0,19504023500,1588417,12,200578000,416,19300248000 95 | gemma:7b-instruct-v1.1-q4_0,16902847417,1587083,13,199503000,361,16700014000 96 | gemma:7b-instruct-v1.1-q4_0,24798591292,1567167,13,196170000,528,24599449000 97 | gemma:7b-instruct-v1.1-q4_0,18411103833,2281250,11,201222000,393,18206021000 98 | gemma:7b-instruct-v1.1-q4_0,20937234625,1614000,14,196298000,447,20737887000 99 | gemma:7b-instruct-v1.1-q4_0,18600305708,1573958,12,201020000,397,18396004000 100 | gemma:7b-instruct-v1.1-q4_0,16262695916,1611208,10,199694000,336,16060522000 101 | gemma:7b-instruct-v1.1-q4_0,17254372584,1236459,10,204083000,367,17048505000 102 | gemma:7b-instruct-v1.1-q4_0,20809218458,1589875,13,200343000,444,20605991000 103 | gemma:7b-instruct-v1.1-q4_0,18005555208,1660458,14,199149000,384,17802922000 104 | gemma:7b-instruct-v1.1-q4_0,14124017209,2165042,13,200990000,302,13919018000 105 | gemma:7b-instruct-v1.1-q4_0,18403512334,1548959,12,200121000,393,18200191000 106 | gemma:7b-instruct-v1.1-q4_0,16022087083,1623458,14,207844000,332,15810752000 107 | gemma:7b-instruct-v1.1-q4_0,19302912250,1774875,10,201694000,398,19097740000 108 | gemma:7b-instruct-v1.1-q4_0,18037223791,1542625,12,200827000,386,17833339000 109 | gemma:7b-instruct-v1.1-q4_0,21166633500,1587125,13,196782000,452,20966706000 110 | gemma:7b-instruct-v1.1-q4_0,13560981959,1569000,15,197901000,291,13359901000 111 | gemma:7b-instruct-v1.1-q4_0,15211939042,1551792,13,204440000,326,15004404000 112 | gemma:7b-instruct-v1.1-q4_0,18415097958,1557375,14,199509000,394,18212406000 113 | gemma:7b-instruct-v1.1-q4_0,14125222791,1571416,14,201387000,303,13920465000 114 | gemma:7b-instruct-v1.1-q4_0,19493995125,1522500,13,200271000,417,19290571000 115 | gemma:7b-instruct-v1.1-q4_0,18374143042,1477208,12,204102000,393,18167038000 116 | gemma:7b-instruct-v1.1-q4_0,18382545625,1660417,14,199214000,393,18179942000 117 | gemma:7b-instruct-v1.1-q4_0,12741432875,1571833,12,203216000,273,12535108000 118 | gemma:7b-instruct-v1.1-q4_0,18045994417,1489417,12,202147000,385,17840828000 119 | gemma:7b-instruct-v1.1-q4_0,20003809000,2144500,15,196734000,428,19803480000 120 | gemma:7b-instruct-v1.1-q4_0,18763574750,1522292,13,202334000,401,18558150000 121 | gemma:7b-instruct-v1.1-q4_0,18186003875,1568292,13,199073000,389,17983943000 122 | mistral:7b-instruct-v0.2-q4_0,9960972584,3706284209,15,150259000,170,6102884000 123 | mistral:7b-instruct-v0.2-q4_0,22614161792,1675584,15,184664000,607,22426613000 124 | mistral:7b-instruct-v0.2-q4_0,14730568834,1833917,11,179017000,400,14548254000 125 | mistral:7b-instruct-v0.2-q4_0,23746488666,1604458,12,178005000,638,23565722000 126 | mistral:7b-instruct-v0.2-q4_0,24065407125,1615291,15,179259000,646,23883289000 127 | mistral:7b-instruct-v0.2-q4_0,9240516833,1571833,12,179746000,252,9057771000 128 | mistral:7b-instruct-v0.2-q4_0,16239092042,2576583,13,180577000,440,16054566000 129 | mistral:7b-instruct-v0.2-q4_0,25136366625,1628000,12,179160000,673,24954383000 130 | mistral:7b-instruct-v0.2-q4_0,18248988041,1834000,12,177433000,493,18068393000 131 | mistral:7b-instruct-v0.2-q4_0,11349872042,1625208,11,179023000,309,11167846000 132 | mistral:7b-instruct-v0.2-q4_0,20190218000,1812500,12,179207000,545,20007928000 133 | mistral:7b-instruct-v0.2-q4_0,5469056709,1602125,9,176357000,149,5289582000 134 | mistral:7b-instruct-v0.2-q4_0,33934389208,967917,12,186754000,898,33745730000 135 | mistral:7b-instruct-v0.2-q4_0,16219015500,1654584,12,176312000,438,16039717000 136 | mistral:7b-instruct-v0.2-q4_0,26467800375,1561250,12,178275000,698,26286815000 137 | mistral:7b-instruct-v0.2-q4_0,18364057083,1642542,11,178011000,496,18183114000 138 | mistral:7b-instruct-v0.2-q4_0,21345310208,1716208,13,178331000,575,21164073000 139 | mistral:7b-instruct-v0.2-q4_0,11939852583,1688042,11,175632000,325,11761099000 140 | mistral:7b-instruct-v0.2-q4_0,4369220750,1642042,9,177575000,119,4188472000 141 | mistral:7b-instruct-v0.2-q4_0,13202522167,1695750,9,173179000,359,13026081000 142 | mistral:7b-instruct-v0.2-q4_0,21374739708,1635541,13,180435000,566,21191464000 143 | mistral:7b-instruct-v0.2-q4_0,23463977625,1546292,14,182927000,629,23278380000 144 | mistral:7b-instruct-v0.2-q4_0,16665205875,1483708,12,180101000,450,16482215000 145 | mistral:7b-instruct-v0.2-q4_0,19282744417,1598000,13,179922000,520,19100067000 146 | mistral:7b-instruct-v0.2-q4_0,3872958750,1499208,13,179371000,105,3690347000 147 | mistral:7b-instruct-v0.2-q4_0,10569405083,1513833,10,179639000,288,10386985000 148 | mistral:7b-instruct-v0.2-q4_0,21086037209,1655417,14,181821000,567,20901296000 149 | mistral:7b-instruct-v0.2-q4_0,21441390625,1615917,13,181802000,564,21256806000 150 | mistral:7b-instruct-v0.2-q4_0,8065194958,1465958,14,181031000,218,7881438000 151 | mistral:7b-instruct-v0.2-q4_0,19904984333,933333,12,184271000,534,19718707000 152 | mistral:7b-instruct-v0.2-q4_0,15352716208,1645125,13,177832000,414,15172015000 153 | mistral:7b-instruct-v0.2-q4_0,14310491875,1620667,13,180657000,387,14127008000 154 | mistral:7b-instruct-v0.2-q4_0,18984646334,1609167,13,181411000,511,18800510000 155 | mistral:7b-instruct-v0.2-q4_0,14768265292,1512125,13,179862000,399,14585602000 156 | mistral:7b-instruct-v0.2-q4_0,17231802500,1680750,13,180035000,465,17048793000 157 | mistral:7b-instruct-v0.2-q4_0,9263438208,1594125,11,178388000,252,9082071000 158 | mistral:7b-instruct-v0.2-q4_0,13468932208,1567666,11,183573000,365,13282493000 159 | mistral:7b-instruct-v0.2-q4_0,16057448875,1494750,15,182875000,434,15871796000 160 | mistral:7b-instruct-v0.2-q4_0,27629485334,1570459,14,181205000,736,27445539000 161 | mistral:7b-instruct-v0.2-q4_0,17247229875,1534458,12,180666000,461,17063583000 162 | wizardlm2:7b-q4_0,24078358583,552515417,44,198941000,621,23325685000 163 | wizardlm2:7b-q4_0,36023245875,1757125,16,179664000,946,35840326000 164 | wizardlm2:7b-q4_0,31262482208,2716958,12,176219000,827,31082179000 165 | wizardlm2:7b-q4_0,44841590292,1670167,13,177061000,1164,44661643000 166 | wizardlm2:7b-q4_0,44792137000,1635375,16,180386000,1150,44591282000 167 | wizardlm2:7b-q4_0,22272390166,1636458,13,177068000,595,22088368000 168 | wizardlm2:7b-q4_0,37864606375,1634917,14,179338000,989,37681620000 169 | wizardlm2:7b-q4_0,35146433042,1659958,13,178852000,921,34964323000 170 | wizardlm2:7b-q4_0,36300727417,1504875,13,176906000,954,36120569000 171 | wizardlm2:7b-q4_0,39351442625,2209333,12,176539000,1013,39171330000 172 | wizardlm2:7b-q4_0,33286751750,1598708,13,179364000,877,33104283000 173 | wizardlm2:7b-q4_0,22370755750,1684500,10,174691000,599,22192628000 174 | wizardlm2:7b-q4_0,43420898542,826209,13,188376000,1131,43230606000 175 | wizardlm2:7b-q4_0,34085431750,1638708,13,176368000,897,33905634000 176 | wizardlm2:7b-q4_0,42615266208,1005333,13,186438000,1107,42426838000 177 | wizardlm2:7b-q4_0,30571044541,1580916,12,176821000,806,30390970000 178 | wizardlm2:7b-q4_0,37628466208,1601833,14,178226000,983,37446908000 179 | wizardlm2:7b-q4_0,35283365291,1695208,12,176963000,924,35103308000 180 | wizardlm2:7b-q4_0,27416755000,2092209,10,174607000,726,27238541000 181 | wizardlm2:7b-q4_0,29268737209,1516292,10,178626000,773,29087030000 182 | wizardlm2:7b-q4_0,40550480333,1632666,14,178376000,1056,40368881000 183 | wizardlm2:7b-q4_0,45914809000,1598791,15,179000000,1188,45732584000 184 | wizardlm2:7b-q4_0,33924926375,1653583,13,181849000,888,33739736000 185 | wizardlm2:7b-q4_0,37477353791,1548916,14,178403000,981,37295740000 186 | wizardlm2:7b-q4_0,26989848250,1541375,14,178347000,689,26808278000 187 | wizardlm2:7b-q4_0,38521145708,1749958,11,178928000,988,38338220000 188 | wizardlm2:7b-q4_0,35635016834,1666000,15,175513000,922,35456138000 189 | wizardlm2:7b-q4_0,53681314625,1641667,14,182473000,1354,53495752000 190 | wizardlm2:7b-q4_0,20869488167,1647375,15,175417000,551,20690785000 191 | wizardlm2:7b-q4_0,47188549875,1785250,13,180876000,1215,47004443000 192 | wizardlm2:7b-q4_0,45083805000,1648000,14,179028000,1161,44901523000 193 | wizardlm2:7b-q4_0,27205898583,1727625,14,174443000,725,27028021000 194 | wizardlm2:7b-q4_0,40608322666,1633916,14,175959000,1029,40429026000 195 | wizardlm2:7b-q4_0,27583382000,1723792,14,178036000,721,27401838000 196 | wizardlm2:7b-q4_0,35475106209,1519667,14,180375000,923,35291619000 197 | wizardlm2:7b-q4_0,31196354750,2108666,12,174770000,791,31018025000 198 | wizardlm2:7b-q4_0,47201852459,1700584,12,175561000,1190,47023033000 199 | wizardlm2:7b-q4_0,37348300000,1711750,16,175789000,968,37169231000 200 | wizardlm2:7b-q4_0,43314354708,1578667,15,179158000,1230,43132215000 201 | wizardlm2:7b-q4_0,22589739000,1600084,13,112445000,1013,22474081000 202 | -------------------------------------------------------------------------------- /benchmarks/M1MAX/benchmarks_prompt_eval.csv: -------------------------------------------------------------------------------- 1 | model,total_duration,load_duration,prompt_eval_count,prompt_eval_duration,eval_count,eval_duration 2 | phi3:3.8b-mini-instruct-4k-q4_K_M,5760859250,424517666,153,279008000,297,5055571000 3 | phi3:3.8b-mini-instruct-4k-q4_K_M,2177963125,1556792,397,681799000,86,1491479000 4 | phi3:3.8b-mini-instruct-4k-q4_K_M,4366949875,1524750,298,517170000,221,3845855000 5 | phi3:3.8b-mini-instruct-4k-q4_K_M,28755436584,1524459,468,784575000,1370,27967443000 6 | phi3:3.8b-mini-instruct-4k-q4_K_M,6273706250,1662000,389,711679000,281,5556896000 7 | phi3:3.8b-mini-instruct-4k-q4_K_M,15750853125,1450417,243,444126000,805,15302680000 8 | phi3:3.8b-mini-instruct-4k-q4_K_M,13998655041,1001166,274,524596000,702,13471352000 9 | phi3:3.8b-mini-instruct-4k-q4_K_M,6333348459,1727750,298,567317000,312,5761493000 10 | phi3:3.8b-mini-instruct-4k-q4_K_M,32389683042,1583417,531,1027330000,1412,31357408000 11 | phi3:3.8b-mini-instruct-4k-q4_K_M,13811370208,1597417,286,529851000,685,13276887000 12 | phi3:3.8b-mini-instruct-4k-q4_K_M,12665833916,1475541,300,576888000,622,12084326000 13 | phi3:3.8b-mini-instruct-4k-q4_K_M,7927568250,1299500,278,529952000,372,7393412000 14 | phi3:3.8b-mini-instruct-4k-q4_K_M,10273220459,1519792,311,592834000,498,9675935000 15 | phi3:3.8b-mini-instruct-4k-q4_K_M,15146337375,1297250,524,1078604000,692,14063883000 16 | phi3:3.8b-mini-instruct-4k-q4_K_M,15426005750,1600333,521,1042035000,699,14380209000 17 | phi3:3.8b-mini-instruct-4k-q4_K_M,15060438166,1802250,440,849435000,682,14205885000 18 | phi3:3.8b-mini-instruct-4k-q4_K_M,7069682417,931709,532,1086934000,307,5979044000 19 | phi3:3.8b-mini-instruct-4k-q4_K_M,16737123916,915833,505,1036836000,738,15697162000 20 | phi3:3.8b-mini-instruct-4k-q4_K_M,17378576791,1608541,492,914597000,796,16458914000 21 | phi3:3.8b-mini-instruct-4k-q4_K_M,9815775458,1703833,495,947919000,446,8861966000 22 | phi3:3.8b-mini-instruct-4k-q4_K_M,18222951083,771250,532,1103083000,818,17116587000 23 | phi3:3.8b-mini-instruct-4k-q4_K_M,16332971292,1151709,336,685198000,765,15644215000 24 | phi3:3.8b-mini-instruct-4k-q4_K_M,14346650667,911500,456,972353000,649,13371034000 25 | phi3:3.8b-mini-instruct-4k-q4_K_M,15732024459,1647167,545,1090579000,699,14635899000 26 | phi3:3.8b-mini-instruct-4k-q4_K_M,8154341666,865916,526,1074272000,354,7076905000 27 | phi3:3.8b-mini-instruct-4k-q4_K_M,8501886750,1511042,255,496721000,419,8000934000 28 | phi3:3.8b-mini-instruct-4k-q4_K_M,10735726666,2035958,255,488175000,531,10242896000 29 | phi3:3.8b-mini-instruct-4k-q4_K_M,4089715167,871625,268,560626000,186,3526339000 30 | phi3:3.8b-mini-instruct-4k-q4_K_M,5966188291,1724208,274,524955000,287,5437171000 31 | phi3:3.8b-mini-instruct-4k-q4_K_M,4484574375,1615041,251,491692000,210,3988396000 32 | phi3:3.8b-mini-instruct-4k-q4_K_M,15746612709,1696042,304,571726000,752,15170438000 33 | phi3:3.8b-mini-instruct-4k-q4_K_M,6444046000,1659000,227,491342000,323,5948410000 34 | phi3:3.8b-mini-instruct-4k-q4_K_M,10338042500,1682250,277,528578000,506,9808128000 35 | phi3:3.8b-mini-instruct-4k-q4_K_M,10203608916,961500,168,360354000,521,9843007000 36 | phi3:3.8b-mini-instruct-4k-q4_K_M,9113553709,960292,216,443574000,463,8668433000 37 | phi3:3.8b-mini-instruct-4k-q4_K_M,6798835042,1592542,214,432712000,344,6362524000 38 | phi3:3.8b-mini-instruct-4k-q4_K_M,20692133584,1583000,242,466391000,1007,20222039000 39 | phi3:3.8b-mini-instruct-4k-q4_K_M,6638285375,1492375,155,297821000,349,6336574000 40 | phi3:3.8b-mini-instruct-4k-q4_K_M,11200697208,1572500,177,350700000,562,10846043000 41 | phi3:3.8b-mini-instruct-4k-q4_K_M,15003989208,1094542,187,365579000,747,14635767000 42 | llama3:8b-instruct-q4_0,11650493833,6205025250,143,424737000,181,5017448000 43 | llama3:8b-instruct-q4_0,9406093750,1763625,327,1418330000,311,7980118000 44 | llama3:8b-instruct-q4_0,7522657417,1695292,269,932146000,264,6584394000 45 | llama3:8b-instruct-q4_0,13277271334,1671709,370,1335780000,452,11935349000 46 | llama3:8b-instruct-q4_0,10499630250,1716959,335,1120785000,357,9372462000 47 | llama3:8b-instruct-q4_0,7067763666,916875,222,749159000,249,6315187000 48 | llama3:8b-instruct-q4_0,10115727417,859292,239,917070000,361,9195122000 49 | llama3:8b-instruct-q4_0,8934573125,862375,243,861377000,316,8069711000 50 | llama3:8b-instruct-q4_0,8873327875,1643791,454,1548869000,282,7316560000 51 | llama3:8b-instruct-q4_0,6112000875,1560750,238,871364000,215,5234653000 52 | llama3:8b-instruct-q4_0,7844720792,1691667,260,1095257000,271,6741981000 53 | llama3:8b-instruct-q4_0,10307235708,1558291,256,932475000,364,9368559000 54 | llama3:8b-instruct-q4_0,9661485625,1303458,261,984549000,340,8671564000 55 | llama3:8b-instruct-q4_0,11060716667,1631125,432,1433163000,370,9620541000 56 | llama3:8b-instruct-q4_0,10645244000,1443542,418,1373282000,355,9265849000 57 | llama3:8b-instruct-q4_0,8788666417,1264459,366,1181290000,296,7602666000 58 | llama3:8b-instruct-q4_0,15332136834,1643750,433,1458473000,527,13866318000 59 | llama3:8b-instruct-q4_0,9841522541,1770125,426,1496358000,318,8339407000 60 | llama3:8b-instruct-q4_0,10860200333,781541,411,1339640000,366,9516142000 61 | llama3:8b-instruct-q4_0,9138854083,1534708,404,1291090000,301,7841675000 62 | llama3:8b-instruct-q4_0,9275490500,1567792,423,1497490000,304,7771464000 63 | llama3:8b-instruct-q4_0,7586795667,1530792,283,941463000,265,6637512000 64 | llama3:8b-instruct-q4_0,10349222625,1620458,378,1411609000,340,8931257000 65 | llama3:8b-instruct-q4_0,9252519833,1984375,463,1487419000,303,7758503000 66 | llama3:8b-instruct-q4_0,10682786458,1640166,441,1424251000,348,9251892000 67 | llama3:8b-instruct-q4_0,8533994375,811416,214,816612000,290,7714024000 68 | llama3:8b-instruct-q4_0,6377794917,1637792,197,767615000,232,5604098000 69 | llama3:8b-instruct-q4_0,6723739250,1098667,238,915034000,228,5804822000 70 | llama3:8b-instruct-q4_0,5019448667,1867084,254,870555000,172,4142749000 71 | llama3:8b-instruct-q4_0,10577897958,989500,211,710456000,383,9863822000 72 | llama3:8b-instruct-q4_0,7085177250,1478417,246,795411000,255,6283892000 73 | llama3:8b-instruct-q4_0,6344199000,1623167,197,752847000,233,5585430000 74 | llama3:8b-instruct-q4_0,12999913083,1015958,217,744267000,483,12250769000 75 | llama3:8b-instruct-q4_0,5928518167,1530959,161,584087000,223,5339122000 76 | llama3:8b-instruct-q4_0,9875117959,958792,179,632231000,371,9239247000 77 | llama3:8b-instruct-q4_0,7910405708,888750,186,624091000,293,7283175000 78 | llama3:8b-instruct-q4_0,11245872208,1700417,223,735429000,423,10504133000 79 | llama3:8b-instruct-q4_0,6677032417,1504959,128,398901000,260,6272653000 80 | llama3:8b-instruct-q4_0,9964620375,825000,140,532730000,384,9428792000 81 | llama3:8b-instruct-q4_0,5417707625,1700959,153,513372000,208,4898470000 82 | gemma:7b-instruct-v1.1-q4_0,10593235708,5190681875,142,473936000,211,4924483000 83 | gemma:7b-instruct-v1.1-q4_0,10425273416,1728291,310,890656000,349,9528694000 84 | gemma:7b-instruct-v1.1-q4_0,11803957875,781125,262,905467000,335,10895285000 85 | gemma:7b-instruct-v1.1-q4_0,13935254250,964583,379,1462465000,377,12469488000 86 | gemma:7b-instruct-v1.1-q4_0,9724872292,857417,323,1451479000,255,8270069000 87 | gemma:7b-instruct-v1.1-q4_0,10542789083,1615333,217,759021000,339,9778711000 88 | gemma:7b-instruct-v1.1-q4_0,9589244959,807959,248,843989000,310,8742297000 89 | gemma:7b-instruct-v1.1-q4_0,7122497959,979000,232,812375000,230,6306822000 90 | gemma:7b-instruct-v1.1-q4_0,11911593250,1515208,445,1408065000,371,10498754000 91 | gemma:7b-instruct-v1.1-q4_0,8328519000,815375,236,834038000,257,7490317000 92 | gemma:7b-instruct-v1.1-q4_0,7983042750,755250,262,964757000,240,7015085000 93 | gemma:7b-instruct-v1.1-q4_0,8113792875,1475750,248,809855000,266,7299474000 94 | gemma:7b-instruct-v1.1-q4_0,9830204750,1588250,267,891219000,324,8934987000 95 | gemma:7b-instruct-v1.1-q4_0,12326161291,1519291,416,1328629000,390,10991153000 96 | gemma:7b-instruct-v1.1-q4_0,7566638792,1688542,444,1405907000,221,6153355000 97 | gemma:7b-instruct-v1.1-q4_0,8884202958,1673708,361,1180120000,278,7697196000 98 | gemma:7b-instruct-v1.1-q4_0,13639745750,841709,439,1572563000,424,12063441000 99 | gemma:7b-instruct-v1.1-q4_0,12274405584,1575000,418,1566624000,377,10700778000 100 | gemma:7b-instruct-v1.1-q4_0,7432701750,946250,408,1439195000,216,5989523000 101 | gemma:7b-instruct-v1.1-q4_0,9200186542,2251834,390,1549128000,272,7644941000 102 | gemma:7b-instruct-v1.1-q4_0,9715648625,1565708,410,1500334000,290,8208879000 103 | gemma:7b-instruct-v1.1-q4_0,9194057583,754875,280,1010755000,291,8179904000 104 | gemma:7b-instruct-v1.1-q4_0,9881420166,2076041,369,1326777000,302,8549148000 105 | gemma:7b-instruct-v1.1-q4_0,9903785000,1551875,447,1554843000,293,8343207000 106 | gemma:7b-instruct-v1.1-q4_0,11827960458,1558625,430,1544787000,362,10277478000 107 | gemma:7b-instruct-v1.1-q4_0,8188960042,825667,204,778744000,264,7407371000 108 | gemma:7b-instruct-v1.1-q4_0,7826196791,1454541,190,695932000,256,7125584000 109 | gemma:7b-instruct-v1.1-q4_0,6673632959,829542,248,948633000,209,5722123000 110 | gemma:7b-instruct-v1.1-q4_0,9284111333,1599917,261,1096911000,291,8181986000 111 | gemma:7b-instruct-v1.1-q4_0,6040638083,1533708,202,824551000,194,5210510000 112 | gemma:7b-instruct-v1.1-q4_0,7439235917,1636375,238,976744000,233,6457721000 113 | gemma:7b-instruct-v1.1-q4_0,9151699417,1523834,189,734831000,299,8411782000 114 | gemma:7b-instruct-v1.1-q4_0,9522489250,832334,212,831392000,309,8687910000 115 | gemma:7b-instruct-v1.1-q4_0,11048425875,1635917,160,588725000,359,10454417000 116 | gemma:7b-instruct-v1.1-q4_0,9985989500,1594708,181,652134000,331,9328363000 117 | gemma:7b-instruct-v1.1-q4_0,5946931125,1568792,169,675372000,183,5266709000 118 | gemma:7b-instruct-v1.1-q4_0,11589978292,1575167,225,942450000,374,10641242000 119 | gemma:7b-instruct-v1.1-q4_0,10390143333,884958,125,456323000,348,9931248000 120 | gemma:7b-instruct-v1.1-q4_0,10461506708,1589125,144,584618000,349,9872370000 121 | gemma:7b-instruct-v1.1-q4_0,9209927125,710417,151,605411000,301,8602058000 122 | mistral:7b-instruct-v0.2-q4_0,10270691000,4762239542,150,423254000,267,5082357000 123 | mistral:7b-instruct-v0.2-q4_0,11137692166,1689458,368,958640000,452,10174285000 124 | mistral:7b-instruct-v0.2-q4_0,5688285500,1593959,294,890002000,219,4793550000 125 | mistral:7b-instruct-v0.2-q4_0,13334664166,1628041,429,1236775000,521,12093503000 126 | mistral:7b-instruct-v0.2-q4_0,4852552708,1507250,370,1246702000,165,3600971000 127 | mistral:7b-instruct-v0.2-q4_0,9074230250,1511708,234,775140000,360,8294675000 128 | mistral:7b-instruct-v0.2-q4_0,7992328666,1127875,267,909769000,312,7079602000 129 | mistral:7b-instruct-v0.2-q4_0,9887086875,1585792,288,948678000,386,8933958000 130 | mistral:7b-instruct-v0.2-q4_0,8064626083,1508458,488,1587056000,280,6472498000 131 | mistral:7b-instruct-v0.2-q4_0,4030901208,1564000,254,887038000,147,3138775000 132 | mistral:7b-instruct-v0.2-q4_0,16704802875,785208,273,839545000,667,15862716000 133 | mistral:7b-instruct-v0.2-q4_0,4499364459,1455084,265,905256000,164,3588956000 134 | mistral:7b-instruct-v0.2-q4_0,11663269792,1506000,283,888620000,460,10770355000 135 | mistral:7b-instruct-v0.2-q4_0,3764058000,2090542,479,1539617000,100,2218713000 136 | mistral:7b-instruct-v0.2-q4_0,10332668000,942792,488,1506045000,368,8822525000 137 | mistral:7b-instruct-v0.2-q4_0,16794252541,910375,412,1420166000,643,15370372000 138 | mistral:7b-instruct-v0.2-q4_0,4131463375,967916,483,1678481000,112,2449662000 139 | mistral:7b-instruct-v0.2-q4_0,11529475250,2251667,473,1384825000,418,10140028000 140 | mistral:7b-instruct-v0.2-q4_0,7205746000,1573666,447,1424816000,253,5775738000 141 | mistral:7b-instruct-v0.2-q4_0,4189374542,1774583,472,1700794000,113,2483348000 142 | mistral:7b-instruct-v0.2-q4_0,11463357708,1667958,489,1492497000,411,9965483000 143 | mistral:7b-instruct-v0.2-q4_0,10653756958,1581333,308,1032181000,407,9616018000 144 | mistral:7b-instruct-v0.2-q4_0,4666658833,852666,424,1444498000,144,3219041000 145 | mistral:7b-instruct-v0.2-q4_0,11394657667,790833,521,1814127000,395,9577369000 146 | mistral:7b-instruct-v0.2-q4_0,4665143125,1441500,497,1620697000,135,3040604000 147 | mistral:7b-instruct-v0.2-q4_0,4841953500,1333292,234,811261000,175,4027493000 148 | mistral:7b-instruct-v0.2-q4_0,8600427750,1444875,235,824757000,333,7772303000 149 | mistral:7b-instruct-v0.2-q4_0,6512478333,1459166,263,945392000,248,5562297000 150 | mistral:7b-instruct-v0.2-q4_0,6582750792,782750,275,1046212000,244,5534037000 151 | mistral:7b-instruct-v0.2-q4_0,10649082542,859917,234,882534000,412,9763990000 152 | mistral:7b-instruct-v0.2-q4_0,2388915209,1500667,267,920789000,67,1463822000 153 | mistral:7b-instruct-v0.2-q4_0,2775571625,843667,208,673336000,96,2099763000 154 | mistral:7b-instruct-v0.2-q4_0,9265896666,1482666,249,894706000,355,8366837000 155 | mistral:7b-instruct-v0.2-q4_0,6803938125,1390791,165,653807000,270,6146155000 156 | mistral:7b-instruct-v0.2-q4_0,10046854875,1581292,195,800722000,395,9241915000 157 | mistral:7b-instruct-v0.2-q4_0,4078527792,1558500,204,733743000,153,3340809000 158 | mistral:7b-instruct-v0.2-q4_0,8841442875,1511458,238,806225000,342,8030885000 159 | mistral:7b-instruct-v0.2-q4_0,9313523333,1723292,141,543207000,377,8766324000 160 | mistral:7b-instruct-v0.2-q4_0,17433306042,1529542,157,540601000,710,16888714000 161 | mistral:7b-instruct-v0.2-q4_0,5367307208,1473917,167,617960000,212,4745393000 162 | wizardlm2:7b-q4_0,17333536083,2655040875,179,487544000,639,14188892000 163 | wizardlm2:7b-q4_0,14422831667,1842625,369,1180346000,542,13238383000 164 | wizardlm2:7b-q4_0,20448082875,1606541,295,1144945000,789,19298144000 165 | wizardlm2:7b-q4_0,27209632875,1570709,430,1435586000,1017,25769815000 166 | wizardlm2:7b-q4_0,20262904583,1483208,371,1219254000,769,19039883000 167 | wizardlm2:7b-q4_0,19109032083,1504500,235,837342000,736,18267420000 168 | wizardlm2:7b-q4_0,21023198375,1606959,268,909548000,815,20109404000 169 | wizardlm2:7b-q4_0,21530378708,1633542,289,1035412000,828,20490613000 170 | wizardlm2:7b-q4_0,20184880541,1474000,489,1702543000,730,18477382000 171 | wizardlm2:7b-q4_0,17222702500,1576750,255,848310000,666,16369652000 172 | wizardlm2:7b-q4_0,22911451291,1502958,274,941719000,877,21965071000 173 | wizardlm2:7b-q4_0,22522063125,1577125,266,947384000,865,21570044000 174 | wizardlm2:7b-q4_0,24626800416,1550916,284,947828000,927,23674449000 175 | wizardlm2:7b-q4_0,19192494750,926333,480,1636571000,691,17552622000 176 | wizardlm2:7b-q4_0,25248877583,1555500,489,1688289000,911,23556054000 177 | wizardlm2:7b-q4_0,18932786875,1649458,413,1385180000,693,17542395000 178 | wizardlm2:7b-q4_0,16373119833,920542,484,1732953000,575,14636696000 179 | wizardlm2:7b-q4_0,19715009375,1574333,474,1629910000,709,18080420000 180 | wizardlm2:7b-q4_0,25425016083,1624083,448,1479377000,876,23940418000 181 | wizardlm2:7b-q4_0,25414132542,939792,473,1868303000,829,23542770000 182 | wizardlm2:7b-q4_0,24909637708,1625541,490,1684661000,893,23219710000 183 | wizardlm2:7b-q4_0,24748196833,1736916,309,1075415000,927,23668112000 184 | wizardlm2:7b-q4_0,21277974500,2265583,425,1500304000,774,19773016000 185 | wizardlm2:7b-q4_0,19264734542,1638167,522,1927352000,665,17331779000 186 | wizardlm2:7b-q4_0,19436649542,1675625,498,1724763000,679,17705122000 187 | wizardlm2:7b-q4_0,25518706917,1517792,235,883738000,954,24630362000 188 | wizardlm2:7b-q4_0,22295134208,1644583,236,1015700000,843,21274928000 189 | wizardlm2:7b-q4_0,13079149583,1500833,264,965163000,492,12109534000 190 | wizardlm2:7b-q4_0,12413921875,1539500,276,980470000,463,11429239000 191 | wizardlm2:7b-q4_0,16976129375,1516167,235,868736000,645,16103604000 192 | wizardlm2:7b-q4_0,23694559375,1535875,268,974944000,887,22715612000 193 | wizardlm2:7b-q4_0,16308496417,1578083,209,773580000,621,15530651000 194 | wizardlm2:7b-q4_0,19391262167,1060500,250,893442000,729,18494395000 195 | wizardlm2:7b-q4_0,14756748792,808667,166,714700000,557,14039518000 196 | wizardlm2:7b-q4_0,19952767834,1536709,196,780259000,760,19168310000 197 | wizardlm2:7b-q4_0,18550410542,1496458,205,769320000,707,17777069000 198 | wizardlm2:7b-q4_0,27004744542,1524167,239,880881000,1013,26119580000 199 | wizardlm2:7b-q4_0,25433941333,1520750,142,557156000,979,24872726000 200 | wizardlm2:7b-q4_0,24380234375,1639750,158,563181000,937,23812909000 201 | wizardlm2:7b-q4_0,25975622833,1523292,168,647975000,977,25323642000 202 | -------------------------------------------------------------------------------- /benchmarks/A6000/benchmarks_eval.csv: -------------------------------------------------------------------------------- 1 | model,total_duration,load_duration,prompt_eval_count,prompt_eval_duration,eval_count,eval_duration 2 | llama3:8b-instruct-q4_0,4818040758,3060090,11,101360000,378,4579798000 3 | llama3:8b-instruct-q4_0,7679704868,2672670,16,80343000,605,7461193000 4 | llama3:8b-instruct-q4_0,7463549478,3018530,12,80624000,590,7244671000 5 | llama3:8b-instruct-q4_0,6336098018,2349520,13,81365000,501,6117827000 6 | llama3:8b-instruct-q4_0,7552222728,2771990,14,80969000,596,7335063000 7 | llama3:8b-instruct-q4_0,6964689788,2858940,13,80412000,549,6747374000 8 | llama3:8b-instruct-q4_0,7644027598,2674010,13,80564000,604,7428711000 9 | llama3:8b-instruct-q4_0,6787025988,3399370,11,80231000,538,6572076000 10 | llama3:8b-instruct-q4_0,6792680168,2691040,13,81170000,537,6573720000 11 | llama3:8b-instruct-q4_0,6387686398,2642430,12,80562000,502,6171225000 12 | llama3:8b-instruct-q4_0,7505147378,3237130,13,81410000,591,7288620000 13 | llama3:8b-instruct-q4_0,6000796788,2921450,10,81199000,471,5782949000 14 | llama3:8b-instruct-q4_0,7728612188,2762490,12,81346000,612,7512529000 15 | llama3:8b-instruct-q4_0,6287167718,2766330,13,81022000,497,6071026000 16 | llama3:8b-instruct-q4_0,8266707238,2978710,13,81780000,651,8047834000 17 | llama3:8b-instruct-q4_0,6348137568,2897840,12,81623000,500,6131721000 18 | llama3:8b-instruct-q4_0,6619820428,3014610,14,81781000,522,6402257000 19 | llama3:8b-instruct-q4_0,8191353147,2792730,12,81634000,646,7973363000 20 | llama3:8b-instruct-q4_0,5239073109,3060640,10,81185000,412,5022644000 21 | llama3:8b-instruct-q4_0,4174530419,2975020,10,82101000,327,3957368000 22 | llama3:8b-instruct-q4_0,6517238098,3228600,13,81555000,510,6300325000 23 | llama3:8b-instruct-q4_0,7876544337,2626790,14,81872000,619,7657922000 24 | llama3:8b-instruct-q4_0,5022898869,2946030,13,82062000,393,4802821000 25 | llama3:8b-instruct-q4_0,7814749247,2904470,13,81732000,615,7597287000 26 | llama3:8b-instruct-q4_0,6060008479,3099400,14,81967000,476,5840477000 27 | llama3:8b-instruct-q4_0,6537684938,3144800,10,82357000,513,6319292000 28 | llama3:8b-instruct-q4_0,5267172648,3065310,12,82059000,411,5047290000 29 | llama3:8b-instruct-q4_0,8716148948,2767650,13,81762000,684,8497659000 30 | llama3:8b-instruct-q4_0,2563882300,3929730,15,83330000,194,2342689000 31 | llama3:8b-instruct-q4_0,6592261258,3000080,13,81776000,514,6373493000 32 | llama3:8b-instruct-q4_0,8506347527,2890710,14,82047000,669,8289012000 33 | llama3:8b-instruct-q4_0,5451001049,3231390,14,82074000,427,5230474000 34 | llama3:8b-instruct-q4_0,7448067122,2875630,13,81873000,584,7228792000 35 | llama3:8b-instruct-q4_0,5527105349,2943160,13,81709000,426,5308780000 36 | llama3:8b-instruct-q4_0,7305679785,2412300,13,82736000,572,7088797000 37 | llama3:8b-instruct-q4_0,4888762493,2641870,12,82556000,382,4671133000 38 | llama3:8b-instruct-q4_0,7689752378,2693410,12,81584000,608,7473629000 39 | llama3:8b-instruct-q4_0,6133777194,2802220,15,82051000,481,5913890000 40 | llama3:8b-instruct-q4_0,8687092486,3034510,14,81991000,677,8469997000 41 | llama3:8b-instruct-q4_0,5855333731,4140240,13,82097000,458,5636337000 42 | gemma:7b-instruct-v1.1-q4_0,9173725231,5181420386,15,86040000,298,3774980000 43 | gemma:7b-instruct-v1.1-q4_0,6189714104,3138640,16,83358000,470,5970300000 44 | gemma:7b-instruct-v1.1-q4_0,4983485904,2736500,11,83038000,375,4764692000 45 | gemma:7b-instruct-v1.1-q4_0,5141066005,2835400,13,83839000,386,4921268000 46 | gemma:7b-instruct-v1.1-q4_0,5962368482,2738390,13,84586000,453,5743621000 47 | gemma:7b-instruct-v1.1-q4_0,5291511537,2689150,12,82908000,401,5074758000 48 | gemma:7b-instruct-v1.1-q4_0,4530812540,2847690,13,83190000,342,4311161000 49 | gemma:7b-instruct-v1.1-q4_0,5187652586,2879400,10,83663000,392,4967351000 50 | gemma:7b-instruct-v1.1-q4_0,5491579508,2911450,13,83377000,416,5273820000 51 | gemma:7b-instruct-v1.1-q4_0,4847147622,2710530,12,83855000,365,4626038000 52 | gemma:7b-instruct-v1.1-q4_0,4480780019,4742480,13,83115000,337,4259372000 53 | gemma:7b-instruct-v1.1-q4_0,5161235206,2890300,10,83341000,393,4943346000 54 | gemma:7b-instruct-v1.1-q4_0,4834612783,2536900,12,82848000,367,4612752000 55 | gemma:7b-instruct-v1.1-q4_0,5126533585,2954620,13,83301000,388,4901467000 56 | gemma:7b-instruct-v1.1-q4_0,6522691947,2450720,13,83361000,496,6302321000 57 | gemma:7b-instruct-v1.1-q4_0,5170973326,2575170,11,83743000,392,4950332000 58 | gemma:7b-instruct-v1.1-q4_0,6048680293,2970690,14,83231000,460,5829416000 59 | gemma:7b-instruct-v1.1-q4_0,5197605965,2838680,12,84369000,392,4976822000 60 | gemma:7b-instruct-v1.1-q4_0,4171000167,2775680,10,82960000,313,3952687000 61 | gemma:7b-instruct-v1.1-q4_0,5076217585,3186970,10,83419000,383,4857475000 62 | gemma:7b-instruct-v1.1-q4_0,5635511969,2989190,13,83307000,425,5416267000 63 | gemma:7b-instruct-v1.1-q4_0,5985764593,2999600,14,84269000,455,5766365000 64 | gemma:7b-instruct-v1.1-q4_0,4379331358,3115950,13,84000000,328,4159015000 65 | gemma:7b-instruct-v1.1-q4_0,5269884096,3155130,12,84421000,399,5050135000 66 | gemma:7b-instruct-v1.1-q4_0,4241613177,3093180,14,83829000,319,4022933000 67 | gemma:7b-instruct-v1.1-q4_0,5569997339,4956890,10,82704000,422,5346831000 68 | gemma:7b-instruct-v1.1-q4_0,5437993808,2896860,12,84205000,411,5217019000 69 | gemma:7b-instruct-v1.1-q4_0,6134175124,3061540,13,83328000,467,5914498000 70 | gemma:7b-instruct-v1.1-q4_0,1046385120,2814590,15,83939000,66,825102000 71 | gemma:7b-instruct-v1.1-q4_0,5284805107,20259641,13,83172000,400,5047624000 72 | gemma:7b-instruct-v1.1-q4_0,5066792004,2930880,14,84587000,383,4846324000 73 | gemma:7b-instruct-v1.1-q4_0,3110259877,3246970,14,83997000,230,2891286000 74 | gemma:7b-instruct-v1.1-q4_0,5707490280,26543770,13,83592000,431,5465486000 75 | gemma:7b-instruct-v1.1-q4_0,4702390462,2592700,12,83233000,355,4484356000 76 | gemma:7b-instruct-v1.1-q4_0,5594202330,3263110,14,83386000,423,5372805000 77 | gemma:7b-instruct-v1.1-q4_0,3159243438,3142420,12,83112000,234,2941181000 78 | gemma:7b-instruct-v1.1-q4_0,4815805673,2911100,12,82490000,363,4595739000 79 | gemma:7b-instruct-v1.1-q4_0,4511576980,3057280,15,84829000,338,4289930000 80 | gemma:7b-instruct-v1.1-q4_0,5395441848,2835320,13,83178000,408,5177560000 81 | gemma:7b-instruct-v1.1-q4_0,5389560628,2902590,13,83066000,409,5168834000 82 | mistral:7b-instruct-v0.2-q4_0,9994888708,8818419088,15,79458000,99,965293000 83 | mistral:7b-instruct-v0.2-q4_0,6690516478,2974940,15,78998000,640,6470695000 84 | mistral:7b-instruct-v0.2-q4_0,3700266083,2834350,11,79525000,350,3485991000 85 | mistral:7b-instruct-v0.2-q4_0,7714572868,2795170,12,79206000,735,7497850000 86 | mistral:7b-instruct-v0.2-q4_0,10298139310,2792300,15,80831000,978,10081668000 87 | mistral:7b-instruct-v0.2-q4_0,3236248189,3049670,12,80035000,299,3021591000 88 | mistral:7b-instruct-v0.2-q4_0,5378284347,2692430,13,78820000,513,5164294000 89 | mistral:7b-instruct-v0.2-q4_0,4541521360,3225660,12,80512000,431,4325453000 90 | mistral:7b-instruct-v0.2-q4_0,5906673232,3216170,12,79125000,565,5691977000 91 | mistral:7b-instruct-v0.2-q4_0,4605615151,3191650,11,79251000,438,4388700000 92 | mistral:7b-instruct-v0.2-q4_0,4210882947,3101340,12,79321000,400,3994643000 93 | mistral:7b-instruct-v0.2-q4_0,2632556993,3147580,9,84365000,243,2410365000 94 | mistral:7b-instruct-v0.2-q4_0,7363766545,2700461,12,79532000,702,7146796000 95 | mistral:7b-instruct-v0.2-q4_0,4869499442,3002870,12,79525000,465,4655471000 96 | mistral:7b-instruct-v0.2-q4_0,5349318587,2823520,12,79175000,510,5128272000 97 | mistral:7b-instruct-v0.2-q4_0,4016059735,4162920,11,79270000,380,3801345000 98 | mistral:7b-instruct-v0.2-q4_0,4185837706,2852920,13,79533000,396,3968836000 99 | mistral:7b-instruct-v0.2-q4_0,4914472333,2494780,11,79410000,464,4699686000 100 | mistral:7b-instruct-v0.2-q4_0,3840828694,4613820,9,79200000,364,3622099000 101 | mistral:7b-instruct-v0.2-q4_0,4115279857,3085210,9,79532000,391,3899502000 102 | mistral:7b-instruct-v0.2-q4_0,6653299879,2852010,13,79560000,634,6438626000 103 | mistral:7b-instruct-v0.2-q4_0,5138783835,2767360,14,79382000,490,4925483000 104 | mistral:7b-instruct-v0.2-q4_0,3728766833,4001590,12,80282000,352,3512309000 105 | mistral:7b-instruct-v0.2-q4_0,4843176403,2921510,13,79562000,462,4626436000 106 | mistral:7b-instruct-v0.2-q4_0,1280692582,2966910,13,79563000,108,1064984000 107 | mistral:7b-instruct-v0.2-q4_0,3249345679,2875610,10,79180000,304,3033629000 108 | mistral:7b-instruct-v0.2-q4_0,4713026931,2758290,14,79430000,445,4499012000 109 | mistral:7b-instruct-v0.2-q4_0,7787452748,2784680,13,79397000,743,7571860000 110 | mistral:7b-instruct-v0.2-q4_0,4513649729,6242290,14,80081000,430,4293210000 111 | mistral:7b-instruct-v0.2-q4_0,6860967660,3120130,12,80112000,655,6644351000 112 | mistral:7b-instruct-v0.2-q4_0,5034143465,2947550,13,79934000,475,4817558000 113 | mistral:7b-instruct-v0.2-q4_0,4003764155,2893640,13,79458000,380,3789382000 114 | mistral:7b-instruct-v0.2-q4_0,8458065005,3113561,13,79314000,806,8243906000 115 | mistral:7b-instruct-v0.2-q4_0,3547345112,3315610,13,79393000,332,3332841000 116 | mistral:7b-instruct-v0.2-q4_0,5267960116,2999660,13,79478000,503,5053587000 117 | mistral:7b-instruct-v0.2-q4_0,4505631329,2592200,11,79203000,428,4290213000 118 | mistral:7b-instruct-v0.2-q4_0,3879039514,2747810,11,79197000,368,3665146000 119 | mistral:7b-instruct-v0.2-q4_0,4511970690,2953700,15,80231000,431,4296686000 120 | mistral:7b-instruct-v0.2-q4_0,7652202637,2149760,14,79393000,731,7439323000 121 | mistral:7b-instruct-v0.2-q4_0,4704698231,2458740,12,78965000,446,4491426000 122 | wizardlm2:7b-q4_0,15070924513,8622678006,44,81532000,616,6233304000 123 | wizardlm2:7b-q4_0,9152603651,2881340,16,80210000,871,8931662000 124 | wizardlm2:7b-q4_0,10544113843,2744450,12,80571000,1000,10329527000 125 | wizardlm2:7b-q4_0,11389486961,2801790,13,80199000,1079,11174087000 126 | wizardlm2:7b-q4_0,9919040567,2704780,16,80425000,939,9702645000 127 | wizardlm2:7b-q4_0,7340690015,2723710,13,79604000,697,7126212000 128 | wizardlm2:7b-q4_0,10750518345,2792360,14,80200000,1018,10533784000 129 | wizardlm2:7b-q4_0,10015145188,2668890,13,79813000,949,9796837000 130 | wizardlm2:7b-q4_0,10923561446,2787610,13,80501000,1035,10707158000 131 | wizardlm2:7b-q4_0,11258601738,2840630,12,80050000,1065,11044367000 132 | wizardlm2:7b-q4_0,9146849930,2964030,13,80923000,868,8927895000 133 | wizardlm2:7b-q4_0,6481992887,2965080,10,79590000,618,6268324000 134 | wizardlm2:7b-q4_0,19981963235,3111680,13,79535000,1875,19768383000 135 | wizardlm2:7b-q4_0,8916448568,2723710,13,79799000,848,8699360000 136 | wizardlm2:7b-q4_0,11276075329,2870870,13,79570000,1060,11058553000 137 | wizardlm2:7b-q4_0,8952258968,3045890,12,79711000,850,8737144000 138 | wizardlm2:7b-q4_0,10816079845,2889790,14,80369000,1023,10600553000 139 | wizardlm2:7b-q4_0,9146620061,2885530,12,79649000,870,8930978000 140 | wizardlm2:7b-q4_0,6372454936,2961740,10,79454000,606,6155580000 141 | wizardlm2:7b-q4_0,9030863170,2484370,10,79622000,858,8816064000 142 | wizardlm2:7b-q4_0,10006705218,2799350,14,80082000,950,9791588000 143 | wizardlm2:7b-q4_0,9777110116,3041700,15,80551000,924,9559888000 144 | wizardlm2:7b-q4_0,9513778154,2659890,13,79983000,903,9297746000 145 | wizardlm2:7b-q4_0,10051248459,3061950,14,80756000,952,9824376000 146 | wizardlm2:7b-q4_0,7328719784,3167480,14,79800000,697,7112179000 147 | wizardlm2:7b-q4_0,10133527319,3022290,11,79895000,959,9918695000 148 | wizardlm2:7b-q4_0,8831328567,3444680,15,80421000,838,8615280000 149 | wizardlm2:7b-q4_0,11800450094,2503230,14,80260000,1109,11586185000 150 | wizardlm2:7b-q4_0,5281884597,2910680,15,80952000,502,5066056000 151 | wizardlm2:7b-q4_0,13684759710,2903070,13,80305000,1288,13460132000 152 | wizardlm2:7b-q4_0,9753628796,2939940,14,81296000,923,9534160000 153 | wizardlm2:7b-q4_0,7296627684,2989600,14,81310000,691,7080262000 154 | wizardlm2:7b-q4_0,10505489443,2782871,14,80931000,992,10288863000 155 | wizardlm2:7b-q4_0,7733299098,2533540,14,80410000,725,7514527000 156 | wizardlm2:7b-q4_0,9664917434,2554840,14,79960000,908,9449462000 157 | wizardlm2:7b-q4_0,9856720207,2912500,12,79832000,933,9639774000 158 | wizardlm2:7b-q4_0,10891222596,2855720,12,79500000,1024,10674402000 159 | wizardlm2:7b-q4_0,10729993304,3076690,16,80020000,1016,10514799000 160 | wizardlm2:7b-q4_0,11338913979,2589760,15,80374000,1067,11121045000 161 | wizardlm2:7b-q4_0,10844689185,2528370,13,81956000,1024,10626723000 162 | llama3:70b-instruct-q4_0,41015265930,13512031709,16,429527000,397,26939433000 163 | llama3:70b-instruct-q4_0,38041141324,2674470,16,424071000,550,37483119000 164 | llama3:70b-instruct-q4_0,35985527107,3008530,12,421819000,520,35427585000 165 | llama3:70b-instruct-q4_0,39225269911,2721400,13,422839000,568,38667181000 166 | llama3:70b-instruct-q4_0,35631456712,5857670,14,424365000,515,35065742000 167 | llama3:70b-instruct-q4_0,38284106173,2744460,13,424212000,554,37723419000 168 | llama3:70b-instruct-q4_0,49630028932,2879810,13,423980000,718,49070782000 169 | llama3:70b-instruct-q4_0,34124342506,5919690,11,422862000,493,33561209000 170 | llama3:70b-instruct-q4_0,35500884451,2823940,13,424057000,514,34941111000 171 | llama3:70b-instruct-q4_0,31403654334,3472691,12,424544000,454,30843901000 172 | llama3:70b-instruct-q4_0,41482368217,2746830,13,424670000,601,40923634000 173 | llama3:70b-instruct-q4_0,42151265030,3078500,10,423265000,610,41591738000 174 | llama3:70b-instruct-q4_0,55278884425,2981080,12,421904000,800,54722702000 175 | llama3:70b-instruct-q4_0,33798508634,2870800,13,424213000,489,33236020000 176 | llama3:70b-instruct-q4_0,48403568606,3044910,13,423754000,700,47843349000 177 | llama3:70b-instruct-q4_0,37730391421,3044500,12,423138000,546,37170886000 178 | llama3:70b-instruct-q4_0,40532298772,2808430,14,424168000,587,39971859000 179 | llama3:70b-instruct-q4_0,41097383415,2622030,12,422746000,596,40538145000 180 | llama3:70b-instruct-q4_0,32845068949,2863260,10,422614000,475,32287214000 181 | llama3:70b-instruct-q4_0,29876649387,3096530,10,422600000,432,29317674000 182 | llama3:70b-instruct-q4_0,36279519975,2606390,13,423634000,524,35719285000 183 | llama3:70b-instruct-q4_0,40648447363,3103170,14,425068000,589,40086936000 184 | llama3:70b-instruct-q4_0,37562415830,2786220,13,424021000,544,37000941000 185 | llama3:70b-instruct-q4_0,41219437045,3208620,13,424025000,597,40660888000 186 | llama3:70b-instruct-q4_0,25523090349,3348160,14,425506000,367,24960359000 187 | llama3:70b-instruct-q4_0,40057111351,4493690,10,422648000,579,39497859000 188 | llama3:70b-instruct-q4_0,24847462385,2872830,12,423605000,358,24287142000 189 | llama3:70b-instruct-q4_0,51782760861,2735030,13,424265000,749,51220545000 190 | llama3:70b-instruct-q4_0,14950771994,2869900,15,424331000,214,14388102000 191 | llama3:70b-instruct-q4_0,33858352404,3234670,13,423982000,489,33298870000 192 | llama3:70b-instruct-q4_0,35035194739,2758150,14,424593000,506,34474194000 193 | llama3:70b-instruct-q4_0,30508694329,2819820,14,424585000,440,29947877000 194 | llama3:70b-instruct-q4_0,40167888825,3195980,13,423812000,582,39606441000 195 | llama3:70b-instruct-q4_0,32433962865,2858160,13,424067000,468,31874792000 196 | llama3:70b-instruct-q4_0,34975528799,2979190,13,425500000,505,34412292000 197 | llama3:70b-instruct-q4_0,30992558538,3150030,12,422074000,448,30433899000 198 | llama3:70b-instruct-q4_0,37719079162,2858520,12,422487000,546,37158995000 199 | llama3:70b-instruct-q4_0,37396285131,3005670,15,425755000,541,36833319000 200 | llama3:70b-instruct-q4_0,43427720531,3211070,14,424435000,629,42867894000 201 | llama3:70b-instruct-q4_0,34977692078,2947900,13,424186000,506,34416077000 202 | command-r:35b-v0.1-q4_0,44215781145,35220986019,12,211585000,228,8649572000 203 | command-r:35b-v0.1-q4_0,20787023446,13324770,14,199068000,530,20440672000 204 | command-r:35b-v0.1-q4_0,28922357607,3137660,9,201383000,735,28586188000 205 | command-r:35b-v0.1-q4_0,10872905175,2813180,11,199574000,277,10535666000 206 | command-r:35b-v0.1-q4_0,33162386229,3497280,12,200339000,839,32826138000 207 | command-r:35b-v0.1-q4_0,19705963070,2760170,10,201745000,501,19369121000 208 | command-r:35b-v0.1-q4_0,16288593513,2660790,11,200425000,415,15954312000 209 | command-r:35b-v0.1-q4_0,18359135533,3262050,8,57428000,470,18161469000 210 | command-r:35b-v0.1-q4_0,25213045928,2864160,11,200260000,639,24878418000 211 | command-r:35b-v0.1-q4_0,20707172856,2880790,10,200730000,526,20369989000 212 | command-r:35b-v0.1-q4_0,13838524680,3065790,11,201021000,352,13501914000 213 | command-r:35b-v0.1-q4_0,16553037345,3151350,8,56977000,423,16359559000 214 | command-r:35b-v0.1-q4_0,15714931810,5454980,10,201454000,401,15372572000 215 | command-r:35b-v0.1-q4_0,22198612664,2988210,11,200245000,563,21860433000 216 | command-r:35b-v0.1-q4_0,23778580532,3091290,11,201368000,605,23439292000 217 | command-r:35b-v0.1-q4_0,26098729643,4702380,9,200572000,662,25761374000 218 | command-r:35b-v0.1-q4_0,29367418090,2956490,12,201145000,745,29030299000 219 | command-r:35b-v0.1-q4_0,22835737087,4281660,10,201003000,580,22496372000 220 | command-r:35b-v0.1-q4_0,18489677655,3057760,8,57870000,474,18294253000 221 | command-r:35b-v0.1-q4_0,11976376101,3997330,8,56180000,307,11782732000 222 | command-r:35b-v0.1-q4_0,19617338190,2912910,11,201254000,498,19279118000 223 | command-r:35b-v0.1-q4_0,30191335124,3340200,13,201293000,766,29848053000 224 | command-r:35b-v0.1-q4_0,13495600869,3215810,11,202331000,343,13157254000 225 | command-r:35b-v0.1-q4_0,23132425208,2480540,10,200707000,589,22797585000 226 | command-r:35b-v0.1-q4_0,15945879692,3569700,12,200355000,406,15607405000 227 | command-r:35b-v0.1-q4_0,8252372102,3121600,8,58475000,212,8056738000 228 | command-r:35b-v0.1-q4_0,22512037485,3518990,10,200822000,572,22174599000 229 | command-r:35b-v0.1-q4_0,16930159616,2694460,12,200749000,430,16592246000 230 | command-r:35b-v0.1-q4_0,12633665864,2532640,13,201305000,322,12295142000 231 | command-r:35b-v0.1-q4_0,21906607262,3276790,11,200365000,557,21568678000 232 | command-r:35b-v0.1-q4_0,18108811153,2648500,12,201242000,461,17771863000 233 | command-r:35b-v0.1-q4_0,20494880284,2684050,12,202195000,521,20155725000 234 | command-r:35b-v0.1-q4_0,22085802622,3154700,12,201964000,561,21748228000 235 | command-r:35b-v0.1-q4_0,20670017836,4575550,11,200925000,526,20331863000 236 | command-r:35b-v0.1-q4_0,24224962545,2823940,12,202519000,616,23886636000 237 | command-r:35b-v0.1-q4_0,12164050350,4898620,10,200562000,309,11825550000 238 | command-r:35b-v0.1-q4_0,20982417405,3020820,10,200930000,535,20646214000 239 | command-r:35b-v0.1-q4_0,26370296464,3140520,13,201245000,671,26032510000 240 | command-r:35b-v0.1-q4_0,31340226255,7048460,11,201449000,795,30997243000 241 | command-r:35b-v0.1-q4_0,17884812384,2880720,11,200265000,455,17549941000 242 | mixtral:8x7b-instruct-v0.1-q4_0,33309013618,28962614497,16,127068000,205,4084129000 243 | mixtral:8x7b-instruct-v0.1-q4_0,9964698464,6397480,15,119174000,481,9705022000 244 | mixtral:8x7b-instruct-v0.1-q4_0,7375288071,2626150,11,107840000,355,7133149000 245 | mixtral:8x7b-instruct-v0.1-q4_0,14233852584,3008040,12,111500000,690,13984597000 246 | mixtral:8x7b-instruct-v0.1-q4_0,12331215623,2893640,15,123522000,596,12072811000 247 | mixtral:8x7b-instruct-v0.1-q4_0,8217248145,3123070,12,113254000,396,7966693000 248 | mixtral:8x7b-instruct-v0.1-q4_0,9370875624,2605410,13,115630000,453,9119425000 249 | mixtral:8x7b-instruct-v0.1-q4_0,7572112294,2974930,12,113545000,364,7320919000 250 | mixtral:8x7b-instruct-v0.1-q4_0,6700351270,3050640,12,113909000,322,6449299000 251 | mixtral:8x7b-instruct-v0.1-q4_0,9213307682,2729861,11,109293000,448,8965645000 252 | mixtral:8x7b-instruct-v0.1-q4_0,9773477541,2960341,12,112197000,473,9525276000 253 | mixtral:8x7b-instruct-v0.1-q4_0,4750628968,2842030,9,100247000,227,4515006000 254 | mixtral:8x7b-instruct-v0.1-q4_0,11494589119,2901120,12,112812000,553,11244561000 255 | mixtral:8x7b-instruct-v0.1-q4_0,8969776708,2935690,12,113787000,432,8721543000 256 | mixtral:8x7b-instruct-v0.1-q4_0,8884281466,4839470,12,113362000,427,8631815000 257 | mixtral:8x7b-instruct-v0.1-q4_0,7986752381,3088980,11,108744000,383,7742485000 258 | mixtral:8x7b-instruct-v0.1-q4_0,8602436422,3636390,13,119342000,415,8345608000 259 | mixtral:8x7b-instruct-v0.1-q4_0,9965283994,3082420,11,112203000,482,9715001000 260 | mixtral:8x7b-instruct-v0.1-q4_0,3561427509,2927590,9,100024000,167,3324357000 261 | mixtral:8x7b-instruct-v0.1-q4_0,9421904485,2994840,9,101028000,457,9185010000 262 | mixtral:8x7b-instruct-v0.1-q4_0,10446110612,3010410,13,116379000,506,10193683000 263 | mixtral:8x7b-instruct-v0.1-q4_0,11472524159,2661630,14,119599000,557,11215377000 264 | mixtral:8x7b-instruct-v0.1-q4_0,9760165331,3088570,12,112534000,472,9505548000 265 | mixtral:8x7b-instruct-v0.1-q4_0,9838153912,3041150,13,115703000,475,9587211000 266 | mixtral:8x7b-instruct-v0.1-q4_0,5992880459,2685790,13,114358000,287,5741061000 267 | mixtral:8x7b-instruct-v0.1-q4_0,8890080066,2686840,10,104781000,431,8649906000 268 | mixtral:8x7b-instruct-v0.1-q4_0,8112064753,2606880,14,122828000,392,7851680000 269 | mixtral:8x7b-instruct-v0.1-q4_0,20041769289,2556660,13,117091000,967,19787377000 270 | mixtral:8x7b-instruct-v0.1-q4_0,4279751840,3108330,14,119144000,200,4024568000 271 | mixtral:8x7b-instruct-v0.1-q4_0,7871446800,2879810,12,113036000,380,7622493000 272 | mixtral:8x7b-instruct-v0.1-q4_0,9766721761,3432460,13,116036000,472,9514724000 273 | mixtral:8x7b-instruct-v0.1-q4_0,7836157279,2868080,13,117390000,376,7581787000 274 | mixtral:8x7b-instruct-v0.1-q4_0,12025593928,5036700,13,117548000,582,11769576000 275 | mixtral:8x7b-instruct-v0.1-q4_0,7576819575,2931350,13,116108000,363,7325006000 276 | mixtral:8x7b-instruct-v0.1-q4_0,8782483174,2689210,13,116157000,424,8528911000 277 | mixtral:8x7b-instruct-v0.1-q4_0,5046357653,3121180,11,107285000,240,4802403000 278 | mixtral:8x7b-instruct-v0.1-q4_0,10678243615,3390340,11,109512000,518,10430937000 279 | mixtral:8x7b-instruct-v0.1-q4_0,10519386413,2642770,15,122941000,508,10261457000 280 | mixtral:8x7b-instruct-v0.1-q4_0,11616504591,2489540,14,120758000,558,11361357000 281 | mixtral:8x7b-instruct-v0.1-q4_0,9529911257,2623010,12,114264000,457,9278363000 282 | -------------------------------------------------------------------------------- /benchmarks/A6000/benchmarks_prompt_eval.csv: -------------------------------------------------------------------------------- 1 | model,total_duration,load_duration,prompt_eval_count,prompt_eval_duration,eval_count,eval_duration 2 | llama3:8b-instruct-q4_0,11985487607,7662449316,143,120298000,331,4065295000 3 | llama3:8b-instruct-q4_0,3039911140,2728880,327,188133000,219,2710294000 4 | llama3:8b-instruct-q4_0,3010060530,3220501,269,160667000,220,2708406000 5 | llama3:8b-instruct-q4_0,4855889110,3075780,370,191252000,357,4523106000 6 | llama3:8b-instruct-q4_0,5536246991,3479330,335,191565000,410,5199185000 7 | llama3:8b-instruct-q4_0,4084227567,3571100,222,141128000,307,3800719000 8 | llama3:8b-instruct-q4_0,4201420339,2676370,239,142412000,316,3917473000 9 | llama3:8b-instruct-q4_0,3457931487,2853410,243,143184000,253,3172430000 10 | llama3:8b-instruct-q4_0,4889996790,3070540,454,262421000,353,4484474000 11 | llama3:8b-instruct-q4_0,2879869837,3181170,238,143954000,210,2593981000 12 | llama3:8b-instruct-q4_0,9495570956,2595560,260,160476000,721,9193745000 13 | llama3:8b-instruct-q4_0,4188451829,2925700,256,146742000,313,3902363000 14 | llama3:8b-instruct-q4_0,4472935004,3209180,261,160670000,336,4173805000 15 | llama3:8b-instruct-q4_0,4517378075,2857670,432,220936000,327,4149943000 16 | llama3:8b-instruct-q4_0,4946892201,2839170,418,217660000,361,4586199000 17 | llama3:8b-instruct-q4_0,4801189139,3782500,366,197852000,351,4457903000 18 | llama3:8b-instruct-q4_0,3965473666,2670780,433,221485000,285,3598984000 19 | llama3:8b-instruct-q4_0,4852161960,2884980,426,220099000,351,4489634000 20 | llama3:8b-instruct-q4_0,4612262216,2905380,411,218908000,335,4250653000 21 | llama3:8b-instruct-q4_0,4305992011,2256280,404,216798000,309,3947946000 22 | llama3:8b-instruct-q4_0,4200343279,2572800,423,221505000,303,3831776000 23 | llama3:8b-instruct-q4_0,4774117558,2637190,283,165087000,356,4466344000 24 | llama3:8b-instruct-q4_0,2760300215,2756470,378,198312000,193,2416156000 25 | llama3:8b-instruct-q4_0,4326105151,4392990,463,264677000,309,3916271000 26 | llama3:8b-instruct-q4_0,4250716230,4665850,441,222859000,306,3878716000 27 | llama3:8b-instruct-q4_0,4458827003,4073040,214,140803000,338,4178553000 28 | llama3:8b-instruct-q4_0,3105794362,2948811,197,139000000,229,2827680000 29 | llama3:8b-instruct-q4_0,3628150600,2859980,238,143767000,269,3343541000 30 | llama3:8b-instruct-q4_0,2734700455,4215510,254,144753000,199,2447161000 31 | llama3:8b-instruct-q4_0,3248589313,3078090,211,140421000,239,2968538000 32 | llama3:8b-instruct-q4_0,3356056415,2688650,246,143005000,248,3071396000 33 | llama3:8b-instruct-q4_0,5518394950,4939270,197,138459000,419,5239171000 34 | llama3:8b-instruct-q4_0,5786808375,4246600,217,139017000,439,5504354000 35 | llama3:8b-instruct-q4_0,3898312394,3194930,161,123112000,295,3633918000 36 | llama3:8b-instruct-q4_0,4827521520,2542060,179,122832000,366,4564147000 37 | llama3:8b-instruct-q4_0,3928763434,2715200,186,124897000,295,3662476000 38 | llama3:8b-instruct-q4_0,6943822164,3075220,223,141115000,531,6662826000 39 | llama3:8b-instruct-q4_0,1846951860,3377080,128,97900000,133,1609167000 40 | llama3:8b-instruct-q4_0,3428897687,3441400,140,119592000,258,3170046000 41 | llama3:8b-instruct-q4_0,5377519468,2843350,153,120808000,413,5115220000 42 | gemma:7b-instruct-v1.1-q4_0,15639782767,11937332626,142,123571000,271,3445629000 43 | gemma:7b-instruct-v1.1-q4_0,4889656241,2866610,310,176172000,355,4571551000 44 | gemma:7b-instruct-v1.1-q4_0,3943256945,2999100,262,171710000,282,3629251000 45 | gemma:7b-instruct-v1.1-q4_0,5210638285,2662680,379,200713000,376,4868517000 46 | gemma:7b-instruct-v1.1-q4_0,3676082990,4042730,323,198618000,259,3335834000 47 | gemma:7b-instruct-v1.1-q4_0,4674252417,2808430,217,153387000,340,4380078000 48 | gemma:7b-instruct-v1.1-q4_0,3416780156,2841890,248,157199000,242,3120458000 49 | gemma:7b-instruct-v1.1-q4_0,3586998859,2614000,232,155459000,257,3286615000 50 | gemma:7b-instruct-v1.1-q4_0,5490303391,4365950,445,224092000,393,5126227000 51 | gemma:7b-instruct-v1.1-q4_0,3100497561,2710450,236,157469000,219,2803375000 52 | gemma:7b-instruct-v1.1-q4_0,3433385616,3662380,262,173126000,244,3119804000 53 | gemma:7b-instruct-v1.1-q4_0,4197720109,3108340,248,157516000,304,3897669000 54 | gemma:7b-instruct-v1.1-q4_0,5944601148,2589830,267,173656000,431,5631244000 55 | gemma:7b-instruct-v1.1-q4_0,5620367262,2682090,416,220126000,404,5258674000 56 | gemma:7b-instruct-v1.1-q4_0,5061869893,4654530,444,223105000,360,4698441000 57 | gemma:7b-instruct-v1.1-q4_0,6720037650,2754030,361,201022000,490,6380656000 58 | gemma:7b-instruct-v1.1-q4_0,4171092228,5657090,439,224753000,293,3805017000 59 | gemma:7b-instruct-v1.1-q4_0,5395706459,3263520,418,222008000,389,5033506000 60 | gemma:7b-instruct-v1.1-q4_0,3778495122,2958800,408,220987000,265,3416520000 61 | gemma:7b-instruct-v1.1-q4_0,3579797958,2470060,390,222544000,248,3219355000 62 | gemma:7b-instruct-v1.1-q4_0,3842593673,2673090,410,221656000,269,3480888000 63 | gemma:7b-instruct-v1.1-q4_0,4046972027,4522531,280,175979000,291,3728989000 64 | gemma:7b-instruct-v1.1-q4_0,5415991209,2646690,369,204466000,391,5071320000 65 | gemma:7b-instruct-v1.1-q4_0,4467205594,5399600,447,223751000,315,4099139000 66 | gemma:7b-instruct-v1.1-q4_0,4476187174,2844820,430,223259000,315,4112413000 67 | gemma:7b-instruct-v1.1-q4_0,5075725384,2578940,204,152567000,370,4783548000 68 | gemma:7b-instruct-v1.1-q4_0,4287765120,3059160,190,127089000,315,4026556000 69 | gemma:7b-instruct-v1.1-q4_0,3205648632,2834410,248,159275000,227,2905031000 70 | gemma:7b-instruct-v1.1-q4_0,4112639827,3964230,261,172345000,296,3799824000 71 | gemma:7b-instruct-v1.1-q4_0,3902545205,3223291,202,153599000,282,3608962000 72 | gemma:7b-instruct-v1.1-q4_0,3240200653,3891390,238,157828000,230,2944254000 73 | gemma:7b-instruct-v1.1-q4_0,4469005204,3344390,189,127349000,329,4204841000 74 | gemma:7b-instruct-v1.1-q4_0,3514088448,3080950,212,154736000,252,3219158000 75 | gemma:7b-instruct-v1.1-q4_0,4421106083,3009510,160,122285000,328,4164310000 76 | gemma:7b-instruct-v1.1-q4_0,5638260232,2959360,181,125143000,422,5378075000 77 | gemma:7b-instruct-v1.1-q4_0,2753941575,3774540,169,123773000,196,2494159000 78 | gemma:7b-instruct-v1.1-q4_0,3343816955,2946090,225,155649000,239,3047019000 79 | gemma:7b-instruct-v1.1-q4_0,4882939620,3564460,125,104873000,365,4641971000 80 | gemma:7b-instruct-v1.1-q4_0,5199458895,2641940,144,122065000,385,4940015000 81 | gemma:7b-instruct-v1.1-q4_0,3312532294,4311060,151,122270000,241,3053649000 82 | mistral:7b-instruct-v0.2-q4_0,4591830845,2977181029,150,116542000,134,1366576000 83 | mistral:7b-instruct-v0.2-q4_0,4536354415,3097930,368,188285000,404,4208444000 84 | mistral:7b-instruct-v0.2-q4_0,3523837938,2681610,294,158996000,315,3226342000 85 | mistral:7b-instruct-v0.2-q4_0,6422116696,2934221,429,212162000,576,6069171000 86 | mistral:7b-instruct-v0.2-q4_0,1966780703,2746341,370,189843000,160,1636145000 87 | mistral:7b-instruct-v0.2-q4_0,4667472357,3856750,234,137736000,426,4394592000 88 | mistral:7b-instruct-v0.2-q4_0,3320805464,3116430,267,153221000,296,3032542000 89 | mistral:7b-instruct-v0.2-q4_0,2109705865,3005730,288,158967000,179,1814673000 90 | mistral:7b-instruct-v0.2-q4_0,3779297892,2779520,488,259715000,322,3379494000 91 | mistral:7b-instruct-v0.2-q4_0,2097554914,4847570,254,142616000,179,1812786000 92 | mistral:7b-instruct-v0.2-q4_0,6367600194,3672780,273,157737000,580,6069963000 93 | mistral:7b-instruct-v0.2-q4_0,3327867264,2929960,265,155914000,297,3033084000 94 | mistral:7b-instruct-v0.2-q4_0,2188579286,4105730,283,158785000,185,1888380000 95 | mistral:7b-instruct-v0.2-q4_0,3294661644,2601150,479,257689000,274,2897161000 96 | mistral:7b-instruct-v0.2-q4_0,5235850756,2546740,488,259393000,455,4837565000 97 | mistral:7b-instruct-v0.2-q4_0,2050762894,3095550,412,214025000,165,1697141000 98 | mistral:7b-instruct-v0.2-q4_0,3997486476,4872580,483,259853000,342,3595814000 99 | mistral:7b-instruct-v0.2-q4_0,2851978487,2507980,473,257157000,233,2453321000 100 | mistral:7b-instruct-v0.2-q4_0,2506546701,2693470,447,215422000,207,2152063000 101 | mistral:7b-instruct-v0.2-q4_0,1280404411,3013770,472,257872000,86,881850000 102 | mistral:7b-instruct-v0.2-q4_0,5054191023,2806550,489,260537000,443,4655391000 103 | mistral:7b-instruct-v0.2-q4_0,4107919017,3765950,308,161280000,369,3805782000 104 | mistral:7b-instruct-v0.2-q4_0,1581691666,2927730,424,213602000,119,1229473000 105 | mistral:7b-instruct-v0.2-q4_0,2246845597,4366370,521,351567000,167,1751289000 106 | mistral:7b-instruct-v0.2-q4_0,4728582588,3219521,497,260686000,411,4327829000 107 | mistral:7b-instruct-v0.2-q4_0,3990111975,3344880,234,137699000,363,3716731000 108 | mistral:7b-instruct-v0.2-q4_0,3759836772,2740270,235,137963000,340,3482933000 109 | mistral:7b-instruct-v0.2-q4_0,2134195115,3475900,263,156945000,182,1840481000 110 | mistral:7b-instruct-v0.2-q4_0,1564255436,2797950,275,155302000,126,1272319000 111 | mistral:7b-instruct-v0.2-q4_0,4751500328,2884700,234,136535000,436,4478034000 112 | mistral:7b-instruct-v0.2-q4_0,5401288949,3661960,267,155461000,495,5109432000 113 | mistral:7b-instruct-v0.2-q4_0,1116860425,2891130,208,135103000,85,846030000 114 | mistral:7b-instruct-v0.2-q4_0,4757585402,2665950,249,140458000,433,4476068000 115 | mistral:7b-instruct-v0.2-q4_0,3268404628,2854250,165,117996000,299,3013521000 116 | mistral:7b-instruct-v0.2-q4_0,1611574444,2771980,195,134667000,134,1342397000 117 | mistral:7b-instruct-v0.2-q4_0,2394952196,2449800,204,134243000,208,2126145000 118 | mistral:7b-instruct-v0.2-q4_0,5289222694,5807470,238,137476000,485,5012494000 119 | mistral:7b-instruct-v0.2-q4_0,7043587368,3156100,141,116582000,659,6787229000 120 | mistral:7b-instruct-v0.2-q4_0,5462184244,2796980,157,117032000,510,5210920000 121 | mistral:7b-instruct-v0.2-q4_0,5364574693,2903970,167,118167000,500,5111446000 122 | wizardlm2:7b-q4_0,8629864741,2597135146,179,118222000,565,5781776000 123 | wizardlm2:7b-q4_0,6974664937,2815000,369,189175000,634,6646127000 124 | wizardlm2:7b-q4_0,10219289826,2859900,295,159741000,944,9920545000 125 | wizardlm2:7b-q4_0,9746254975,5073650,430,214501000,887,9388310000 126 | wizardlm2:7b-q4_0,6709605917,2694450,371,189849000,606,6379144000 127 | wizardlm2:7b-q4_0,8432355161,3846900,235,139727000,781,8152322000 128 | wizardlm2:7b-q4_0,10248107255,3230480,268,156849000,947,9950159000 129 | wizardlm2:7b-q4_0,9602346644,3192560,289,158817000,884,9302846000 130 | wizardlm2:7b-q4_0,7790433889,2611130,489,267452000,696,7380940000 131 | wizardlm2:7b-q4_0,8310775900,3746610,255,142439000,766,8027586000 132 | wizardlm2:7b-q4_0,10267493915,6571590,274,158083000,948,9966987000 133 | wizardlm2:7b-q4_0,8300097680,3746600,266,158192000,763,7999658000 134 | wizardlm2:7b-q4_0,10811084897,2667910,284,160043000,994,10504571000 135 | wizardlm2:7b-q4_0,7919906740,4337040,480,268835000,708,7509441000 136 | wizardlm2:7b-q4_0,9307173643,2674960,489,267695000,838,8899451000 137 | wizardlm2:7b-q4_0,7155931858,2937990,413,215929000,645,6801334000 138 | wizardlm2:7b-q4_0,8237087701,4277890,484,266245000,736,7827773000 139 | wizardlm2:7b-q4_0,9220027393,3030670,474,258094000,830,8820738000 140 | wizardlm2:7b-q4_0,9570967563,2972980,448,218611000,870,9210918000 141 | wizardlm2:7b-q4_0,9476340473,2919140,473,257894000,857,9077919000 142 | wizardlm2:7b-q4_0,8150626580,5568610,490,266713000,729,7739681000 143 | wizardlm2:7b-q4_0,9159880232,4509830,309,162850000,842,8855491000 144 | wizardlm2:7b-q4_0,7719514619,2802840,425,216320000,699,7364252000 145 | wizardlm2:7b-q4_0,8243644411,4188970,522,352318000,731,7750957000 146 | wizardlm2:7b-q4_0,8403240611,2796560,498,268641000,746,7994631000 147 | wizardlm2:7b-q4_0,10602302416,3230900,235,139060000,984,10323682000 148 | wizardlm2:7b-q4_0,8465794411,2896920,236,140080000,785,8185721000 149 | wizardlm2:7b-q4_0,7850779589,3963740,264,156635000,723,7554682000 150 | wizardlm2:7b-q4_0,5188093543,3468920,276,157337000,472,4893676000 151 | wizardlm2:7b-q4_0,8005178680,3558240,235,139887000,741,7727450000 152 | wizardlm2:7b-q4_0,9152160823,2719880,268,156494000,844,8858476000 153 | wizardlm2:7b-q4_0,6969728638,2917240,209,135571000,647,6697172000 154 | wizardlm2:7b-q4_0,7530058789,2761570,250,141425000,690,7249397000 155 | wizardlm2:7b-q4_0,6035194695,3266870,166,119284000,562,5778561000 156 | wizardlm2:7b-q4_0,11732350579,2821140,196,134928000,1094,11459417000 157 | wizardlm2:7b-q4_0,5921996485,3609930,205,135406000,548,5648485000 158 | wizardlm2:7b-q4_0,10102029725,3079840,239,140463000,938,9824755000 159 | wizardlm2:7b-q4_0,9079805672,3613210,142,116634000,850,8824022000 160 | wizardlm2:7b-q4_0,9438924473,3509570,158,118687000,881,9181012000 161 | wizardlm2:7b-q4_0,9578003043,3382310,168,119163000,895,9320746000 162 | llama3:70b-instruct-q4_0,66814671966,51678171438,143,811169000,210,14184369000 163 | llama3:70b-instruct-q4_0,26750122776,6803320,327,1510668000,367,25093290000 164 | llama3:70b-instruct-q4_0,18266884295,2976820,269,1335018000,246,16793091000 165 | llama3:70b-instruct-q4_0,31763639639,2931910,370,1562086000,438,30055710000 166 | llama3:70b-instruct-q4_0,24915318852,3229500,335,1551981000,339,23220005000 167 | llama3:70b-instruct-q4_0,18179989115,3670970,222,1170203000,247,16869383000 168 | llama3:70b-instruct-q4_0,22694887626,2778610,239,1179242000,312,21377099000 169 | llama3:70b-instruct-q4_0,20137599741,3472690,243,1182863000,275,18811944000 170 | llama3:70b-instruct-q4_0,26797016557,2979190,454,2109301000,357,24543770000 171 | llama3:70b-instruct-q4_0,20760087932,2767230,238,1181261000,284,19436281000 172 | llama3:70b-instruct-q4_0,17963569615,6167490,260,1341941000,241,16478037000 173 | llama3:70b-instruct-q4_0,20334966101,2743620,256,1195126000,278,19000336000 174 | llama3:70b-instruct-q4_0,21521063004,3111190,261,1345761000,293,20036644000 175 | llama3:70b-instruct-q4_0,17605110434,2938480,432,1833029000,229,15626951000 176 | llama3:70b-instruct-q4_0,39714559009,3660560,418,1822739000,549,37744522000 177 | llama3:70b-instruct-q4_0,16091780010,4818660,366,1571565000,210,14375438000 178 | llama3:70b-instruct-q4_0,30017120185,3990210,433,1833208000,408,28037753000 179 | llama3:70b-instruct-q4_0,33115250742,4484610,426,1829636000,453,31139550000 180 | llama3:70b-instruct-q4_0,24834604331,4393330,411,1822173000,333,22868483000 181 | llama3:70b-instruct-q4_0,31921664759,3069080,404,1816659000,436,29959677000 182 | llama3:70b-instruct-q4_0,20181128330,3100790,423,1828775000,266,18206333000 183 | llama3:70b-instruct-q4_0,28971256372,3856880,283,1355742000,401,27470966000 184 | llama3:70b-instruct-q4_0,26495432846,4504930,378,1577667000,361,24773040000 185 | llama3:70b-instruct-q4_0,34173931739,2706260,463,2117859000,464,31912034000 186 | llama3:70b-instruct-q4_0,27437643477,4343670,441,1837406000,370,25454574000 187 | llama3:70b-instruct-q4_0,21121143303,2987780,214,1168927000,290,19811356000 188 | llama3:70b-instruct-q4_0,19255580882,3586180,197,1160459000,263,17952851000 189 | llama3:70b-instruct-q4_0,16558594031,3988810,238,1180986000,223,15237736000 190 | llama3:70b-instruct-q4_0,13090309688,2599330,254,1192965000,172,11758323000 191 | llama3:70b-instruct-q4_0,19272367552,2755420,211,1170746000,263,17959830000 192 | llama3:70b-instruct-q4_0,18958494572,3429040,246,1183876000,258,17632204000 193 | llama3:70b-instruct-q4_0,21028628704,2760660,197,1159425000,289,19730093000 194 | llama3:70b-instruct-q4_0,22516838864,3652460,217,1172424000,310,21205125000 195 | llama3:70b-instruct-q4_0,15691604130,3568790,161,853388000,216,14698110000 196 | llama3:70b-instruct-q4_0,36571105883,3295290,179,860875000,519,35570205000 197 | llama3:70b-instruct-q4_0,21815581553,2905370,186,861789000,305,20812776000 198 | llama3:70b-instruct-q4_0,23767378304,3615090,223,1172778000,329,22453741000 199 | llama3:70b-instruct-q4_0,21476831923,3484000,128,655125000,304,20678086000 200 | llama3:70b-instruct-q4_0,21876316544,3027940,140,843516000,307,20891589000 201 | llama3:70b-instruct-q4_0,14771236619,3104560,153,848785000,203,13782354000 202 | command-r:35b-v0.1-q4_0,28159050928,10061979337,138,408860000,452,17552189000 203 | command-r:35b-v0.1-q4_0,21130239473,4057880,315,651524000,515,20338113000 204 | command-r:35b-v0.1-q4_0,24695627526,3918340,264,640824000,608,23913970000 205 | command-r:35b-v0.1-q4_0,10671200077,4043710,381,760987000,251,9768681000 206 | command-r:35b-v0.1-q4_0,15296539139,3353820,333,746473000,366,14404062000 207 | command-r:35b-v0.1-q4_0,11712976818,4499340,216,571341000,283,11000031000 208 | command-r:35b-v0.1-q4_0,11934106097,2813180,244,577226000,288,11218130000 209 | command-r:35b-v0.1-q4_0,18044585252,2880790,234,574510000,442,17328341000 210 | command-r:35b-v0.1-q4_0,13224884248,4078270,448,888826000,309,12191329000 211 | command-r:35b-v0.1-q4_0,4574046943,4057470,237,575851000,100,3851212000 212 | command-r:35b-v0.1-q4_0,15951642760,5808860,258,637944000,388,15169512000 213 | command-r:35b-v0.1-q4_0,19482445672,3874270,248,579340000,479,18761853000 214 | command-r:35b-v0.1-q4_0,18468039451,4811530,263,638882000,452,17684829000 215 | command-r:35b-v0.1-q4_0,8250724375,7153500,433,884498000,184,7216747000 216 | command-r:35b-v0.1-q4_0,24834634476,4335570,442,884072000,600,23805869000 217 | command-r:35b-v0.1-q4_0,18379071722,6043660,370,764927000,443,17465515000 218 | command-r:35b-v0.1-q4_0,3607672712,4828160,449,1018075000,63,2443001000 219 | command-r:35b-v0.1-q4_0,17143532521,2719940,428,880466000,408,16117994000 220 | command-r:35b-v0.1-q4_0,7767627305,3842700,412,874991000,173,6749109000 221 | command-r:35b-v0.1-q4_0,21152068843,4779400,406,871493000,509,20133565000 222 | command-r:35b-v0.1-q4_0,25504065136,4096290,424,880018000,616,24479155000 223 | command-r:35b-v0.1-q4_0,23134908885,2880370,277,642409000,568,22350453000 224 | command-r:35b-v0.1-q4_0,16241828770,2889380,374,765545000,389,15336310000 225 | command-r:35b-v0.1-q4_0,16383732770,5920610,443,887982000,387,15349014000 226 | command-r:35b-v0.1-q4_0,25179984446,4153490,427,879202000,609,24157484000 227 | command-r:35b-v0.1-q4_0,10023215696,3980770,202,568452000,240,9314316000 228 | command-r:35b-v0.1-q4_0,13505606468,4205870,190,426442000,332,12937074000 229 | command-r:35b-v0.1-q4_0,10066223226,2830640,243,578387000,240,9349098000 230 | command-r:35b-v0.1-q4_0,6309536464,4823890,260,638881000,143,5525106000 231 | command-r:35b-v0.1-q4_0,19012665131,4931730,203,566938000,468,18301580000 232 | command-r:35b-v0.1-q4_0,12159197377,4214460,238,576816000,293,11439147000 233 | command-r:35b-v0.1-q4_0,7606669275,3084730,190,425419000,182,7039838000 234 | command-r:35b-v0.1-q4_0,22115936774,3421080,208,568386000,545,21408808000 235 | command-r:35b-v0.1-q4_0,21990799544,3733820,159,417030000,550,21430972000 236 | command-r:35b-v0.1-q4_0,28184831638,3320860,180,422372000,702,27621620000 237 | command-r:35b-v0.1-q4_0,22614850864,3160360,172,420648000,563,22053488000 238 | command-r:35b-v0.1-q4_0,12989938848,6758830,220,573445000,315,12273681000 239 | command-r:35b-v0.1-q4_0,17155282271,3474080,124,309839000,430,16705909000 240 | command-r:35b-v0.1-q4_0,31530835490,3352000,138,412219000,787,30980018000 241 | command-r:35b-v0.1-q4_0,22888003794,2897270,151,414065000,571,22332371000 242 | mixtral:8x7b-instruct-v0.1-q4_0,28933717407,23099726270,151,826799000,243,4873761000 243 | mixtral:8x7b-instruct-v0.1-q4_0,4342501773,2890770,368,1066361000,155,3134816000 244 | mixtral:8x7b-instruct-v0.1-q4_0,6400598139,3037790,294,986569000,260,5271757000 245 | mixtral:8x7b-instruct-v0.1-q4_0,10131164400,2874220,429,1134807000,434,8861056000 246 | mixtral:8x7b-instruct-v0.1-q4_0,9724148638,3352000,370,1063009000,415,8519978000 247 | mixtral:8x7b-instruct-v0.1-q4_0,4644374643,3993970,234,926564000,178,3582338000 248 | mixtral:8x7b-instruct-v0.1-q4_0,6441303669,2737410,267,959500000,262,5341045000 249 | mixtral:8x7b-instruct-v0.1-q4_0,8794230066,2818770,288,985113000,376,7672701000 250 | mixtral:8x7b-instruct-v0.1-q4_0,3217230229,3809530,488,1219320000,91,1855834000 251 | mixtral:8x7b-instruct-v0.1-q4_0,7104180851,3740390,254,948768000,297,6014251000 252 | mixtral:8x7b-instruct-v0.1-q4_0,10547794391,2927230,273,970448000,464,9432383000 253 | mixtral:8x7b-instruct-v0.1-q4_0,7404814331,2723780,265,959566000,312,6305748000 254 | mixtral:8x7b-instruct-v0.1-q4_0,12506165437,2883650,283,976818000,559,11388875000 255 | mixtral:8x7b-instruct-v0.1-q4_0,6712812440,2988700,479,1210061000,262,5364043000 256 | mixtral:8x7b-instruct-v0.1-q4_0,12631928497,3204920,488,1216376000,549,11273368000 257 | mixtral:8x7b-instruct-v0.1-q4_0,14568726813,2947070,412,1119338000,649,13309112000 258 | mixtral:8x7b-instruct-v0.1-q4_0,4119807112,4160060,483,1220258000,135,2755452000 259 | mixtral:8x7b-instruct-v0.1-q4_0,6641228409,3143870,473,1206001000,259,5296012000 260 | mixtral:8x7b-instruct-v0.1-q4_0,6797199050,12319330,447,1163276000,268,5483466000 261 | mixtral:8x7b-instruct-v0.1-q4_0,7571946512,4624150,472,1199525000,303,6224698000 262 | mixtral:8x7b-instruct-v0.1-q4_0,13059095459,2761990,489,1221713000,565,11696734000 263 | mixtral:8x7b-instruct-v0.1-q4_0,8829930126,4879700,308,1000643000,374,7686285000 264 | mixtral:8x7b-instruct-v0.1-q4_0,9115735597,3884330,424,1137234000,383,7835754000 265 | mixtral:8x7b-instruct-v0.1-q4_0,10581819041,4551870,521,1350284000,437,9090182000 266 | mixtral:8x7b-instruct-v0.1-q4_0,7662935423,4840450,497,1235490000,306,6286381000 267 | mixtral:8x7b-instruct-v0.1-q4_0,7982319684,2832030,234,930039000,342,6914370000 268 | mixtral:8x7b-instruct-v0.1-q4_0,3299741570,3736190,235,912891000,112,2250819000 269 | mixtral:8x7b-instruct-v0.1-q4_0,7073301770,2718050,263,955577000,296,5980947000 270 | mixtral:8x7b-instruct-v0.1-q4_0,8563843305,3239910,275,971603000,367,7455959000 271 | mixtral:8x7b-instruct-v0.1-q4_0,3597882440,3174120,234,926137000,126,2534356000 272 | mixtral:8x7b-instruct-v0.1-q4_0,7235670771,3494400,267,957604000,300,6143300000 273 | mixtral:8x7b-instruct-v0.1-q4_0,5724979307,3141500,208,901740000,232,4688301000 274 | mixtral:8x7b-instruct-v0.1-q4_0,11891593135,3468840,249,948017000,530,10804256000 275 | mixtral:8x7b-instruct-v0.1-q4_0,7655152332,2830720,165,844739000,330,6672450000 276 | mixtral:8x7b-instruct-v0.1-q4_0,6986043010,2940430,195,870840000,296,5978227000 277 | mixtral:8x7b-instruct-v0.1-q4_0,6064876847,3185010,204,885607000,249,5043745000 278 | mixtral:8x7b-instruct-v0.1-q4_0,9278317667,2699060,238,923169000,402,8219047000 279 | mixtral:8x7b-instruct-v0.1-q4_0,9151204227,2899710,141,770874000,409,8245524000 280 | mixtral:8x7b-instruct-v0.1-q4_0,14523405123,2959850,157,836567000,664,13551891000 281 | mixtral:8x7b-instruct-v0.1-q4_0,17047984951,4032880,167,812629000,789,16096602000 282 | -------------------------------------------------------------------------------- /benchmarks/M1MAX/plot_eval.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | 2024-04-30T16:46:19.341707 10 | image/svg+xml 11 | 12 | 13 | Matplotlib v3.8.4, https://matplotlib.org/ 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 31 | 32 | 33 | 34 | 41 | 42 | 43 | 44 | 45 | 48 | 49 | 50 | 51 | 52 | 53 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 88 | 89 | 90 | 91 | 92 | 93 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 116 | 117 | 118 | 119 | 120 | 121 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 157 | 158 | 159 | 160 | 161 | 162 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 206 | 207 | 208 | 209 | 210 | 211 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 241 | 242 | 243 | 244 | 245 | 246 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 304 | 326 | 341 | 366 | 387 | 423 | 430 | 431 | 432 | 433 | 434 | 435 | 436 | 437 | 438 | 439 | 440 | 441 | 442 | 443 | 444 | 445 | 446 | 447 | 448 | 476 | 496 | 509 | 522 | 529 | 569 | 595 | 602 | 633 | 649 | 671 | 693 | 719 | 726 | 741 | 759 | 760 | 761 | 762 | 763 | 764 | 765 | 766 | 767 | 768 | 769 | 770 | 771 | 772 | 773 | 774 | 775 | 776 | 777 | 778 | 779 | 780 | 781 | 782 | 783 | 784 | 785 | 786 | 787 | 788 | 789 | 790 | 791 | 792 | 793 | 794 | 795 | 796 | 797 | 798 | 799 | 800 | 801 | 808 | 850 | 851 | 852 | 853 | 854 | 855 | 856 | 857 | 858 | 859 | 860 | 861 | 862 | 863 | 864 | 865 | 866 | 867 | 868 | 869 | 870 | 871 | 872 | 873 | 874 | 875 | 876 | 877 | 878 | 879 | 880 | 881 | 882 | 916 | 930 | 942 | 943 | 944 | 945 | 946 | 947 | 948 | 949 | 950 | 951 | 952 | 953 | 954 | 955 | 956 | 957 | 958 | 959 | 960 | 961 | 962 | 963 | 964 | 965 | 966 | 967 | 968 | 969 | 970 | 971 | 972 | 973 | 974 | 975 | 976 | 977 | 978 | 979 | 980 | 981 | 982 | 983 | 984 | 985 | 986 | 987 | 988 | 989 | 990 | 991 | 992 | 993 | 994 | 995 | 996 | 997 | 998 | 999 | 1000 | 1001 | 1002 | 1003 | 1004 | 1005 | 1006 | 1007 | 1008 | 1009 | 1010 | 1011 | 1012 | 1013 | 1014 | 1035 | 1051 | 1078 | 1079 | 1080 | 1081 | 1082 | 1083 | 1084 | 1085 | 1086 | 1087 | 1088 | 1089 | 1090 | 1091 | 1092 | 1093 | 1094 | 1095 | 1096 | 1097 | 1098 | 1099 | 1100 | 1101 | 1107 | 1108 | 1109 | 1115 | 1116 | 1117 | 1123 | 1124 | 1125 | 1131 | 1132 | 1133 | 1139 | 1140 | 1141 | 1144 | 1145 | 1146 | 1149 | 1150 | 1151 | 1154 | 1155 | 1156 | 1159 | 1160 | 1161 | 1164 | 1165 | 1166 | 1167 | 1168 | 1169 | 1170 | 1171 | 1172 | 1173 | -------------------------------------------------------------------------------- /benchmarks/M1MAX/plot_prompt_eval.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | 2024-04-30T16:46:19.439220 10 | image/svg+xml 11 | 12 | 13 | Matplotlib v3.8.4, https://matplotlib.org/ 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 31 | 32 | 33 | 34 | 41 | 42 | 43 | 44 | 45 | 48 | 49 | 50 | 51 | 52 | 53 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 88 | 89 | 90 | 91 | 92 | 93 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 117 | 118 | 119 | 120 | 121 | 122 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 159 | 160 | 161 | 162 | 163 | 164 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 209 | 210 | 211 | 212 | 213 | 214 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 245 | 246 | 247 | 248 | 249 | 250 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 309 | 331 | 346 | 371 | 392 | 428 | 435 | 436 | 437 | 438 | 439 | 440 | 441 | 442 | 443 | 444 | 445 | 446 | 447 | 448 | 449 | 450 | 451 | 452 | 453 | 481 | 501 | 514 | 527 | 534 | 574 | 600 | 607 | 638 | 654 | 676 | 698 | 724 | 731 | 746 | 764 | 765 | 766 | 767 | 768 | 769 | 770 | 771 | 772 | 773 | 774 | 775 | 776 | 777 | 778 | 779 | 780 | 781 | 782 | 783 | 784 | 785 | 786 | 787 | 788 | 789 | 790 | 791 | 792 | 793 | 794 | 795 | 796 | 797 | 798 | 799 | 800 | 801 | 802 | 803 | 804 | 805 | 806 | 813 | 855 | 856 | 857 | 858 | 859 | 860 | 861 | 862 | 863 | 864 | 865 | 866 | 867 | 868 | 869 | 870 | 871 | 872 | 873 | 874 | 875 | 876 | 877 | 878 | 879 | 880 | 881 | 882 | 883 | 884 | 885 | 886 | 887 | 921 | 935 | 947 | 948 | 949 | 950 | 951 | 952 | 953 | 954 | 955 | 956 | 957 | 958 | 959 | 960 | 961 | 962 | 963 | 964 | 965 | 966 | 967 | 968 | 969 | 970 | 971 | 972 | 973 | 974 | 975 | 976 | 977 | 978 | 979 | 980 | 981 | 982 | 983 | 984 | 985 | 986 | 987 | 988 | 989 | 990 | 991 | 992 | 993 | 994 | 995 | 996 | 997 | 998 | 999 | 1000 | 1001 | 1002 | 1003 | 1004 | 1005 | 1006 | 1007 | 1008 | 1009 | 1010 | 1011 | 1012 | 1013 | 1014 | 1015 | 1016 | 1017 | 1018 | 1019 | 1040 | 1056 | 1083 | 1084 | 1085 | 1086 | 1087 | 1088 | 1089 | 1090 | 1091 | 1092 | 1093 | 1094 | 1095 | 1096 | 1097 | 1098 | 1099 | 1100 | 1101 | 1102 | 1103 | 1104 | 1105 | 1106 | 1112 | 1113 | 1114 | 1120 | 1121 | 1122 | 1128 | 1129 | 1130 | 1136 | 1137 | 1138 | 1144 | 1145 | 1146 | 1149 | 1150 | 1151 | 1154 | 1155 | 1156 | 1159 | 1160 | 1161 | 1164 | 1165 | 1166 | 1169 | 1170 | 1171 | 1172 | 1173 | 1174 | 1175 | 1176 | 1177 | 1178 | -------------------------------------------------------------------------------- /benchmarks/1080TI/plot_prompt_eval.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | 2024-05-01T21:20:59.087513 10 | image/svg+xml 11 | 12 | 13 | Matplotlib v3.8.4, https://matplotlib.org/ 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 31 | 32 | 33 | 34 | 41 | 42 | 43 | 44 | 45 | 48 | 49 | 50 | 51 | 52 | 53 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 88 | 89 | 90 | 91 | 92 | 93 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 130 | 131 | 132 | 133 | 134 | 135 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 166 | 167 | 168 | 169 | 170 | 171 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 216 | 217 | 218 | 219 | 220 | 221 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 | 273 | 274 | 275 | 276 | 277 | 278 | 290 | 291 | 292 | 293 | 294 | 295 | 296 | 297 | 298 | 299 | 300 | 303 | 304 | 305 | 306 | 307 | 308 | 309 | 310 | 311 | 312 | 313 | 314 | 315 | 316 | 317 | 318 | 341 | 363 | 378 | 403 | 424 | 460 | 467 | 468 | 469 | 470 | 471 | 472 | 473 | 474 | 475 | 476 | 477 | 478 | 479 | 480 | 481 | 482 | 483 | 484 | 485 | 513 | 533 | 546 | 579 | 592 | 599 | 625 | 632 | 663 | 679 | 701 | 723 | 749 | 756 | 771 | 789 | 790 | 791 | 792 | 793 | 794 | 795 | 796 | 797 | 798 | 799 | 800 | 801 | 802 | 803 | 804 | 805 | 806 | 807 | 808 | 809 | 810 | 811 | 812 | 813 | 814 | 815 | 816 | 817 | 818 | 819 | 820 | 821 | 822 | 823 | 824 | 825 | 826 | 827 | 828 | 829 | 830 | 831 | 838 | 880 | 881 | 882 | 883 | 884 | 885 | 886 | 887 | 888 | 889 | 890 | 891 | 892 | 893 | 894 | 895 | 896 | 897 | 898 | 899 | 900 | 901 | 902 | 903 | 904 | 905 | 906 | 907 | 908 | 909 | 910 | 911 | 912 | 946 | 960 | 972 | 973 | 974 | 975 | 976 | 977 | 978 | 979 | 980 | 981 | 982 | 983 | 984 | 985 | 986 | 987 | 988 | 989 | 990 | 991 | 992 | 993 | 994 | 995 | 996 | 997 | 998 | 999 | 1000 | 1001 | 1002 | 1003 | 1004 | 1005 | 1006 | 1007 | 1008 | 1009 | 1010 | 1011 | 1012 | 1013 | 1014 | 1015 | 1016 | 1017 | 1018 | 1019 | 1020 | 1021 | 1022 | 1023 | 1024 | 1025 | 1026 | 1027 | 1028 | 1029 | 1030 | 1031 | 1032 | 1033 | 1034 | 1035 | 1036 | 1037 | 1038 | 1039 | 1040 | 1041 | 1042 | 1043 | 1044 | 1065 | 1081 | 1108 | 1109 | 1110 | 1111 | 1112 | 1113 | 1114 | 1115 | 1116 | 1117 | 1118 | 1119 | 1120 | 1121 | 1122 | 1123 | 1124 | 1125 | 1126 | 1127 | 1128 | 1129 | 1130 | 1131 | 1137 | 1138 | 1139 | 1145 | 1146 | 1147 | 1153 | 1154 | 1155 | 1161 | 1162 | 1163 | 1169 | 1170 | 1171 | 1174 | 1175 | 1176 | 1179 | 1180 | 1181 | 1184 | 1185 | 1186 | 1189 | 1190 | 1191 | 1194 | 1195 | 1196 | 1197 | 1198 | 1199 | 1200 | 1201 | 1202 | 1203 | --------------------------------------------------------------------------------