├── .gitignore ├── LICENSE ├── README.md ├── buildkite_benchmarks.xlsx ├── requirements.txt ├── serve.py └── update_data.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dashboard 2 | 3 | vLLM performance dashboard 4 | 5 | ## Install requirements 6 | 7 | ```shell 8 | pip install -r requirements.txt 9 | ``` 10 | 11 | ## (For Maintainers) Run `update_data.py` periodically 12 | 13 | ```shell 14 | export BUILDKIT_API_TOKEN="" 15 | python update_data.py 16 | ``` 17 | 18 | ## (For Users) Serve the data and view it locally 19 | 20 | ```shell 21 | python serve.py 22 | ``` 23 | -------------------------------------------------------------------------------- /buildkite_benchmarks.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vllm-project/dashboard/9d6e04aaf2f04a621df667f7e802c63bd87efb39/buildkite_benchmarks.xlsx -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | flask 2 | pandas 3 | openpyxl 4 | requests 5 | dash 6 | plotly 7 | -------------------------------------------------------------------------------- /serve.py: -------------------------------------------------------------------------------- 1 | import os 2 | import dash 3 | import pandas as pd 4 | import plotly.express as px 5 | from plotly.subplots import make_subplots 6 | from dash import html, dcc, Input, Output, dash_table, clientside_callback 7 | from datetime import datetime 8 | from zoneinfo import ZoneInfo # This is for Python 3.9 and later 9 | from flask import request 10 | 11 | tz = 'America/Los_Angeles' 12 | local_tz = ZoneInfo(tz) # Change this to your timezone, e.g., 'Europe/London', 'Asia/Tokyo' 13 | 14 | # Load your Excel file 15 | file_path = "buildkite_benchmarks.xlsx" 16 | df = pd.read_excel(file_path) 17 | metrics = [col for col in df.columns if df[col].dtype in ['float64', 'int64']] 18 | 19 | timestamp = os.path.getmtime(file_path) 20 | last_modified_time = datetime.fromtimestamp(timestamp).astimezone(local_tz) 21 | readable_time = last_modified_time.strftime('%Y-%m-%d %H:%M:%S') + ' ' + tz 22 | 23 | # Create a column with the commit link as text (since Plotly hover can't render HTML) 24 | df['commit_link'] = df['commit_url'].apply(lambda x: f"URL: {x}") 25 | 26 | # Create a Dash application 27 | app = dash.Dash(__name__) 28 | 29 | def create_metric_figure(metric): 30 | # Create a subplot figure for each metric, as there is only one metric per plot here 31 | fig = make_subplots(rows=1, cols=1, shared_xaxes=True) 32 | trace = px.line( 33 | df, 34 | x='build_datetime', 35 | y=metric, 36 | title=f'Trend of {metric}', 37 | text='commit_link' # This is where the hover text comes from 38 | ).data[0] 39 | fig.add_trace(trace, row=1, col=1) 40 | 41 | fig.update_traces( 42 | mode='markers+lines', 43 | hovertemplate='Time:%{x}
Value: %{y}
' # Customize hover text 44 | ) 45 | fig.update_layout( 46 | height=300, # Fixed height as each metric is in a separate plot now 47 | title_text=f"Performance Metrics for {metric}", 48 | hovermode='closest' 49 | ) 50 | return fig 51 | 52 | # Dropdown to select the metric for plotting 53 | app.layout = html.Div([ 54 | html.H1('Performance Metrics Over Time'), 55 | dcc.Markdown("Click on data point to show commit url.", id='url-click-output'), # Use dcc.Markdown to handle HTML 56 | dcc.Dropdown( 57 | id='metric-select', 58 | options=[{'label': metric, 'value': metric} for metric in metrics], 59 | value=metrics, 60 | multi=True 61 | ), 62 | # Generate a plot for each metric, hidden by default 63 | html.Div([dcc.Graph(id=f'graph-{metric}', figure=create_metric_figure(metric)) 64 | for metric in metrics]), 65 | dcc.Store(id='plot-visibility'), 66 | html.Div(f"Last Updated: {readable_time}", style={'position': 'fixed', 'bottom': '10px', 'right': '10px'}), 67 | ]) 68 | 69 | clientside_callback( 70 | """ 71 | function(selectedMetrics, ...plotIds) { 72 | // Loop through all plot IDs and set visibility 73 | plotIds.forEach((plotId) => { 74 | const plotElement = document.getElementById(plotId); 75 | if (plotElement) { 76 | plotElement.style.display = selectedMetrics.includes(plotId.replace('graph-', '')) ? 'block' : 'none'; 77 | } 78 | }); 79 | return null; 80 | } 81 | """, 82 | Output('plot-visibility', 'data'), 83 | Input('metric-select', 'value'), 84 | state=[Input(f'graph-{metric}', 'id') for metric in metrics] # Passing plot IDs as state 85 | ) 86 | 87 | for metric in metrics: 88 | clientside_callback( 89 | """ 90 | function(clickData) { 91 | if (clickData) { 92 | var url = clickData.points[0].text.split("URL: ")[1]; 93 | return `[Open Commit ${url}](${url})`; 94 | } 95 | return "Click on a point to see the URL."; 96 | } 97 | """, 98 | Output('url-click-output', 'children', allow_duplicate=True), 99 | Input(f'graph-{metric}', 'clickData'), 100 | prevent_initial_call=True, 101 | ) 102 | 103 | # Flask request logging 104 | @app.server.before_request 105 | def log_request(): 106 | # This will print the request details before each request is processed. 107 | if request.path != '/_reload-hash': 108 | print(f"Request received at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") 109 | print(f"Request path: {request.path}") 110 | print(f"Request method: {request.method}") 111 | print(f"Request headers: {request.headers}") 112 | print(f"Request body: {request.get_data(as_text=True)}") 113 | 114 | if __name__ == '__main__': 115 | app.run_server(debug=True) 116 | -------------------------------------------------------------------------------- /update_data.py: -------------------------------------------------------------------------------- 1 | import os 2 | import requests 3 | from datetime import datetime, timedelta 4 | 5 | import subprocess 6 | 7 | def run_curl_command(token, url): 8 | command = [ 9 | "curl", 10 | "-H", f"Authorization: Bearer {token}", 11 | "-X", "GET", 12 | url 13 | ] 14 | 15 | try: 16 | result = subprocess.run(command, check=True, text=True, capture_output=True) 17 | return result.stdout # Returns the content of the log file 18 | except subprocess.CalledProcessError as e: 19 | raise Exception(f"Command failed with return code {e.returncode}: {e.output}") 20 | 21 | 22 | def get_builds(org_slug, branch, token, days=30): 23 | url = f"https://api.buildkite.com/v2/organizations/{org_slug}/builds" 24 | headers = { 25 | "Authorization": f"Bearer {token}", 26 | "Content-Type": "application/json" 27 | } 28 | 29 | # Calculate the date 30 days ago from today 30 | date_from = (datetime.utcnow() - timedelta(days=days)).isoformat() + "Z" 31 | 32 | params = { 33 | "branch": branch, 34 | "created_from": date_from, 35 | "per_page": "100", 36 | } 37 | 38 | all_builds = [] 39 | while url: 40 | response = requests.get(url, headers=headers, params=params) 41 | if response.status_code == 200: 42 | all_builds.extend(response.json()) 43 | # Parse the Link header and look for a 'next' relation 44 | link_header = response.headers.get('Link', None) 45 | url = None 46 | if link_header: 47 | links = link_header.split(',') 48 | next_link = [link for link in links if 'rel="next"' in link] 49 | if next_link: 50 | next_url = next_link[0].split(';')[0].strip('<>') 51 | url = next_url 52 | params = {} # Clear params because next URL will have necessary params 53 | else: 54 | raise Exception(f"Failed to get builds: {response.status_code} - {response.text}") 55 | 56 | return all_builds 57 | 58 | import re 59 | 60 | # Define a list of dictionaries for patterns 61 | log_patterns = [ 62 | {'key': 'Average Latency', 'pattern': re.compile(r"Avg latency: ([\d.]+) seconds")}, 63 | {'key': '10% Percentile Latency', 'pattern': re.compile(r"10% percentile latency: ([\d.]+) seconds")}, 64 | {'key': '25% Percentile Latency', 'pattern': re.compile(r"25% percentile latency: ([\d.]+) seconds")}, 65 | {'key': '50% Percentile Latency', 'pattern': re.compile(r"50% percentile latency: ([\d.]+) seconds")}, 66 | {'key': '75% Percentile Latency', 'pattern': re.compile(r"75% percentile latency: ([\d.]+) seconds")}, 67 | {'key': '90% Percentile Latency', 'pattern': re.compile(r"90% percentile latency: ([\d.]+) seconds")}, 68 | {'key': 'Throughput', 'pattern': re.compile(r"Throughput: ([\d.]+) requests/s")}, 69 | {'key': 'Token Throughput', 'pattern': re.compile(r"Throughput: [\d.]+ requests/s, ([\d.]+) tokens/s")}, 70 | {'key': 'Successful Requests', 'pattern': re.compile(r"Successful requests: +(\d+)")}, 71 | {'key': 'Benchmark Duration', 'pattern': re.compile(r"Benchmark duration \(s\): +([\d.]+)")}, 72 | {'key': 'Total Input Tokens', 'pattern': re.compile(r"Total input tokens: +(\d+)")}, 73 | {'key': 'Total Generated Tokens', 'pattern': re.compile(r"Total generated tokens: +(\d+)")}, 74 | {'key': 'Request Throughput', 'pattern': re.compile(r"Request throughput \(req/s\): +([\d.]+)")}, 75 | {'key': 'Input Token Throughput', 'pattern': re.compile(r"Input token throughput \(tok/s\): +([\d.]+)")}, 76 | {'key': 'Output Token Throughput', 'pattern': re.compile(r"Output token throughput \(tok/s\): +([\d.]+)")}, 77 | {'key': 'Mean TTFT', 'pattern': re.compile(r"Mean TTFT \(ms\): +([\d.]+)")}, 78 | {'key': 'Median TTFT', 'pattern': re.compile(r"Median TTFT \(ms\): +([\d.]+)")}, 79 | {'key': 'P99 TTFT', 'pattern': re.compile(r"P99 TTFT \(ms\): +([\d.]+)")}, 80 | {'key': 'Mean TPOT', 'pattern': re.compile(r"Mean TPOT \(ms\): +([\d.]+)")}, 81 | {'key': 'Median TPOT', 'pattern': re.compile(r"Median TPOT \(ms\): +([\d.]+)")}, 82 | {'key': 'P99 TPOT', 'pattern': re.compile(r"P99 TPOT \(ms\): +([\d.]+)")} 83 | ] 84 | 85 | # Function to process log entries using defined patterns 86 | def extract_data_from_logs(logs, patterns=log_patterns): 87 | results = {} 88 | for line in logs.split('\n'): 89 | for pattern_dict in patterns: 90 | match = pattern_dict['pattern'].search(line) 91 | if match: 92 | results[pattern_dict['key']] = match.group(1) 93 | return results 94 | 95 | # Replace 'your_token_here' with your actual Buildkite API token 96 | API_TOKEN = os.environ.get("BUILDKIT_API_TOKEN") # or 'your_token_here' 97 | ORG_SLUG = "vllm" # Replace 'vllm' with the actual slug of your organization if different 98 | BRANCH = "main" 99 | cache_dir = ".cache" 100 | os.makedirs(cache_dir, exist_ok=True) 101 | 102 | columns = [ 103 | 'commit', 104 | 'commit_url', 105 | 'build_datetime', 106 | 'Average Latency', 107 | '10% Percentile Latency', 108 | '25% Percentile Latency', 109 | '50% Percentile Latency', 110 | '75% Percentile Latency', 111 | '90% Percentile Latency', 112 | 'Throughput', 113 | 'Token Throughput', 114 | 'Successful Requests', 115 | 'Benchmark Duration', 116 | 'Total Input Tokens', 117 | 'Total Generated Tokens', 118 | 'Request Throughput', 119 | 'Input Token Throughput', 120 | 'Output Token Throughput', 121 | 'Mean TTFT', 122 | 'Median TTFT', 123 | 'P99 TTFT', 124 | 'Mean TPOT', 125 | 'Median TPOT', 126 | 'P99 TPOT' 127 | ] 128 | values = [] 129 | 130 | builds = get_builds(ORG_SLUG, BRANCH, API_TOKEN) 131 | for build in builds: 132 | commit = build['commit'] 133 | commit_url = f"{build['pipeline']['repository'].replace('.git', '')}/commit/{build['commit']}" 134 | raw_log_url = None 135 | for job in build.get('jobs', []): 136 | if 'name' in job and job['name'] == "Benchmarks": 137 | raw_log_url = job['raw_log_url'] 138 | break 139 | if raw_log_url is None: 140 | continue 141 | build_datetime = build['created_at'] 142 | filename = f"{build_datetime}_{commit}.log" 143 | filepath = os.path.join(cache_dir, filename) 144 | if os.path.exists(filepath): 145 | print(f"Skipping downloading {filepath} for commit {commit} because it already exists") 146 | else: 147 | data = run_curl_command(API_TOKEN, raw_log_url) 148 | if len(data) <= 100: 149 | print(f"Skipping processing {filepath} for commit {commit} because the log is empty") 150 | continue 151 | with open(filepath, "w") as f: 152 | f.write(data) 153 | print(f"Saved {filepath} for commit {commit}") 154 | with open(filepath, "r") as f: 155 | logs = f.read() 156 | results = extract_data_from_logs(logs) 157 | values.append([commit, commit_url, build_datetime] + [results.get(col, "") for col in columns[3:]]) 158 | 159 | import pandas as pd 160 | df = pd.DataFrame(values, columns=columns) 161 | df.to_excel("buildkite_benchmarks.xlsx", index=False) --------------------------------------------------------------------------------