├── .github └── workflows │ ├── ci.yml │ ├── python-publish.yml │ ├── run-tests.yml │ └── update-prices.yml ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.md ├── pricing_table.md ├── pyproject.toml ├── tach.yml ├── tests └── test_costs.py ├── tokencost.png ├── tokencost ├── __init__.py ├── constants.py ├── costs.py └── model_prices.json ├── tox.ini └── update_prices.py /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Tach Check 2 | 3 | on: [pull_request] 4 | 5 | jobs: 6 | tach-check: 7 | runs-on: ubuntu-latest 8 | 9 | steps: 10 | - name: Checkout code 11 | uses: actions/checkout@v2 12 | 13 | - name: Set up Python 14 | uses: actions/setup-python@v2 15 | with: 16 | python-version: '3.x' 17 | 18 | - name: Install dependencies 19 | run: | 20 | python -m pip install --upgrade pip 21 | pip install tach==0.6.9 22 | 23 | - name: Run Tach 24 | run: tach check -------------------------------------------------------------------------------- /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflow will upload a Python Package using Twine when a release is created 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries 3 | 4 | # This workflow uses actions that are not certified by GitHub. 5 | # They are provided by a third-party and are governed by 6 | # separate terms of service, privacy policy, and support 7 | # documentation. 8 | 9 | name: Upload Python Package 10 | 11 | on: 12 | release: 13 | types: [published] 14 | 15 | permissions: 16 | contents: read 17 | 18 | jobs: 19 | deploy: 20 | 21 | runs-on: ubuntu-latest 22 | 23 | steps: 24 | - uses: actions/checkout@v3 25 | - name: Set up Python 26 | uses: actions/setup-python@v3 27 | with: 28 | python-version: '3.x' 29 | - name: Install dependencies 30 | run: | 31 | python -m pip install --upgrade pip 32 | pip install build 33 | - name: Build package 34 | run: python -m build 35 | - name: Publish package 36 | uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 37 | with: 38 | user: __token__ 39 | password: ${{ secrets.PYPI_API_TOKEN }} 40 | -------------------------------------------------------------------------------- /.github/workflows/run-tests.yml: -------------------------------------------------------------------------------- 1 | name: Run Tests 2 | 3 | on: 4 | pull_request: 5 | branches: [ main ] 6 | push: 7 | branches: [ main ] 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | python-version: ['3.10', '3.11'] 15 | 16 | steps: 17 | - uses: actions/checkout@v3 18 | 19 | - name: Set up Python ${{ matrix.python-version }} 20 | uses: actions/setup-python@v4 21 | with: 22 | python-version: ${{ matrix.python-version }} 23 | 24 | - name: Install dependencies 25 | run: | 26 | python -m pip install --upgrade pip 27 | python -m pip install -e ".[dev]" 28 | 29 | - name: Run tests 30 | run: | 31 | python -m pytest tests 32 | env: 33 | # This handles the Anthropic tests that require an API key 34 | # Uses a dummy key for tests since they're skipped without a key 35 | ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY || 'dummy-key-for-testing' }} -------------------------------------------------------------------------------- /.github/workflows/update-prices.yml: -------------------------------------------------------------------------------- 1 | name: Daily Price Updates 2 | 3 | on: 4 | schedule: 5 | - cron: '0 0 * * *' # Run at midnight UTC daily 6 | workflow_dispatch: # Allow manual trigger 7 | 8 | jobs: 9 | update-prices: 10 | runs-on: ubuntu-latest 11 | permissions: 12 | contents: write 13 | pull-requests: write 14 | 15 | steps: 16 | - uses: actions/checkout@v4 17 | 18 | - name: Set up Python 19 | uses: actions/setup-python@v4 20 | with: 21 | python-version: '3.11' 22 | 23 | - name: Install dependencies 24 | run: | 25 | python -m pip install --upgrade pip 26 | pip install -e ".[dev]" 27 | 28 | - name: Update prices 29 | run: python update_prices.py 30 | 31 | - name: Check for changes 32 | id: git-check 33 | run: | 34 | git status --porcelain 35 | echo "changed=$(git status --porcelain | wc -l)" >> $GITHUB_OUTPUT 36 | 37 | - name: Get current date 38 | id: date 39 | run: echo "date=$(date +'%d-%m-%Y')" >> $GITHUB_OUTPUT 40 | 41 | - name: Show status 42 | run: | 43 | if [ "${{ steps.git-check.outputs.changed }}" -gt "0" ]; then 44 | echo "Changes detected - will create PR" 45 | git status 46 | else 47 | echo "No changes detected - skipping PR creation" 48 | fi 49 | 50 | - name: Create Pull Request 51 | if: steps.git-check.outputs.changed > 0 52 | uses: peter-evans/create-pull-request@v5 53 | with: 54 | commit-message: "chore: Update token prices (${{ steps.date.outputs.date }})" 55 | title: "chore: Daily token price update (${{ steps.date.outputs.date }})" 56 | body: | 57 | GitHub Action to update token prices 58 | 59 | This PR updates the token prices based on the latest data. 60 | 61 | Changes detected: 62 | ``` 63 | ${{ steps.git-check.outputs.changed }} files modified 64 | ``` 65 | branch: automated/price-updates-${{ steps.date.outputs.date }} 66 | base: main 67 | delete-branch: true 68 | draft: false 69 | reviewers: | 70 | the-praxs 71 | labels: | 72 | automated 73 | prices -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # VSCode 10 | .vscode/ 11 | 12 | # Distribution / packaging 13 | .Python 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | share/python-wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | MANIFEST 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .nox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | *.cover 52 | *.py,cover 53 | .hypothesis/ 54 | .pytest_cache/ 55 | cover/ 56 | 57 | # Translations 58 | *.mo 59 | *.pot 60 | 61 | # Django stuff: 62 | *.log 63 | local_settings.py 64 | db.sqlite3 65 | db.sqlite3-journal 66 | 67 | # Flask stuff: 68 | instance/ 69 | .webassets-cache 70 | 71 | # Scrapy stuff: 72 | .scrapy 73 | 74 | # Sphinx documentation 75 | docs/_build/ 76 | 77 | # PyBuilder 78 | .pybuilder/ 79 | target/ 80 | 81 | # Jupyter Notebook 82 | .ipynb_checkpoints 83 | 84 | # IPython 85 | profile_default/ 86 | ipython_config.py 87 | 88 | # pyenv 89 | # For a library or package, you might want to ignore these files since the code is 90 | # intended to run in multiple environments; otherwise, check them in: 91 | # .python-version 92 | 93 | # pipenv 94 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 95 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 96 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 97 | # install all needed dependencies. 98 | #Pipfile.lock 99 | 100 | # poetry 101 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 102 | # This is especially recommended for binary packages to ensure reproducibility, and is more 103 | # commonly ignored for libraries. 104 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 105 | #poetry.lock 106 | 107 | # pdm 108 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 109 | #pdm.lock 110 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 111 | # in version control. 112 | # https://pdm.fming.dev/#use-with-ide 113 | .pdm.toml 114 | 115 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 116 | __pypackages__/ 117 | 118 | # Celery stuff 119 | celerybeat-schedule 120 | celerybeat.pid 121 | 122 | # SageMath parsed files 123 | *.sage.py 124 | 125 | # Environments 126 | .env 127 | .venv 128 | env/ 129 | venv/ 130 | ENV/ 131 | env.bak/ 132 | venv.bak/ 133 | 134 | # Spyder project settings 135 | .spyderproject 136 | .spyproject 137 | 138 | # Rope project settings 139 | .ropeproject 140 | 141 | # mkdocs documentation 142 | /site 143 | 144 | # mypy 145 | .mypy_cache/ 146 | .dmypy.json 147 | dmypy.json 148 | 149 | # Pyre type checker 150 | .pyre/ 151 | 152 | # pytype static type analyzer 153 | .pytype/ 154 | 155 | # Cython debug symbols 156 | cython_debug/ 157 | 158 | # PyCharm 159 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 160 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 161 | # and can be added to the global gitignore or merged into this file. For a more nuclear 162 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 163 | #.idea/ 164 | 165 | .secrets 166 | .DS_STORE 167 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 AgentOps 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include model_prices.json -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=61.0", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.setuptools] 6 | include-package-data = true 7 | 8 | [tool.setuptools.package-data] 9 | tokencost = ["model_prices.json"] 10 | 11 | [project] 12 | name = "tokencost" 13 | version = "0.1.22" 14 | authors = [ 15 | { name = "Trisha Pan", email = "trishaepan@gmail.com" }, 16 | { name = "Alex Reibman", email = "areibman@gmail.com" }, 17 | { name = "Pratyush Shukla", email = "ps4534@nyu.edu" }, 18 | ] 19 | description = "To calculate token and translated USD cost of string and message calls to OpenAI, for example when used by AI agents" 20 | readme = "README.md" 21 | requires-python = ">=3.10" 22 | classifiers = [ 23 | "Programming Language :: Python :: 3", 24 | "License :: OSI Approved :: MIT License", 25 | "Operating System :: OS Independent", 26 | ] 27 | dependencies = [ 28 | "tiktoken>=0.9.0", 29 | "aiohttp>=3.9.3", 30 | "anthropic>=0.34.0" 31 | ] 32 | 33 | [project.optional-dependencies] 34 | dev = [ 35 | "pytest>=7.4.4", 36 | "flake8>=3.1.0", 37 | "coverage[toml]>=7.4.0", 38 | "tach>=0.6.9", 39 | "tabulate>=0.9.0", 40 | "pandas>=2.1.0", 41 | "python-dotenv>=1.0.0", 42 | ] 43 | 44 | [project.urls] 45 | Homepage = "https://github.com/AgentOps-AI/tokencost" 46 | Issues = "https://github.com/AgentOps-AI/tokencost/issues" 47 | -------------------------------------------------------------------------------- /tach.yml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=https://raw.githubusercontent.com/gauge-sh/tach/v0.6.9/public/tach-yml-schema.json 2 | modules: 3 | - path: tokencost 4 | depends_on: 5 | - tokencost.constants 6 | - tokencost.costs 7 | - path: tokencost.constants 8 | depends_on: [] 9 | - path: tokencost.costs 10 | depends_on: 11 | - tokencost.constants 12 | - path: update_prices 13 | depends_on: 14 | - tokencost 15 | exclude: 16 | - .*__pycache__ 17 | - .*egg-info 18 | - docs 19 | - tests 20 | - venv 21 | source_root: . 22 | -------------------------------------------------------------------------------- /tests/test_costs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import os 5 | import pytest 6 | from decimal import Decimal 7 | from dotenv import load_dotenv 8 | 9 | # Load environment variables for ANTHROPIC_API_KEY 10 | load_dotenv() 11 | 12 | from tokencost.costs import ( 13 | count_message_tokens, 14 | count_string_tokens, 15 | calculate_cost_by_tokens, 16 | calculate_prompt_cost, 17 | calculate_completion_cost, 18 | ) 19 | 20 | # 15 tokens 21 | MESSAGES = [ 22 | {"role": "user", "content": "Hello"}, 23 | {"role": "assistant", "content": "Hi there!"}, 24 | ] 25 | 26 | MESSAGES_WITH_NAME = [ 27 | {"role": "user", "content": "Hello", "name": "John"}, 28 | {"role": "assistant", "content": "Hi there!"}, 29 | ] 30 | 31 | # 4 tokens 32 | STRING = "Hello, world!" 33 | 34 | 35 | # Chat models only, no embeddings (such as ada) since embeddings only does strings, not messages 36 | @pytest.mark.parametrize( 37 | "model,expected_output", 38 | [ 39 | ("gpt-3.5-turbo", 15), 40 | ("gpt-3.5-turbo-0301", 17), 41 | ("gpt-3.5-turbo-0613", 15), 42 | ("gpt-3.5-turbo-16k", 15), 43 | ("gpt-3.5-turbo-16k-0613", 15), 44 | ("gpt-3.5-turbo-1106", 15), 45 | ("gpt-3.5-turbo-instruct", 15), 46 | ("gpt-4", 15), 47 | ("gpt-4-0314", 15), 48 | ("gpt-4-0613", 15), 49 | ("gpt-4-32k", 15), 50 | ("gpt-4-32k-0314", 15), 51 | ("gpt-4-1106-preview", 15), 52 | ("gpt-4-vision-preview", 15), 53 | ("gpt-4o", 15), 54 | ("azure/gpt-4o", 15), 55 | pytest.param("claude-3-opus-latest", 11, 56 | marks=pytest.mark.skipif( 57 | bool(os.getenv("ANTHROPIC_API_KEY")), 58 | reason="ANTHROPIC_API_KEY environment variable not set" 59 | )), 60 | ], 61 | ) 62 | def test_count_message_tokens(model, expected_output): 63 | print(model) 64 | assert count_message_tokens(MESSAGES, model) == expected_output 65 | 66 | 67 | # Chat models only, no embeddings 68 | @pytest.mark.parametrize( 69 | "model,expected_output", 70 | [ 71 | ("gpt-3.5-turbo", 17), 72 | ("gpt-3.5-turbo-0301", 17), 73 | ("gpt-3.5-turbo-0613", 17), 74 | ("gpt-3.5-turbo-1106", 17), 75 | ("gpt-3.5-turbo-instruct", 17), 76 | ("gpt-3.5-turbo-16k", 17), 77 | ("gpt-3.5-turbo-16k-0613", 17), 78 | ("gpt-4", 17), 79 | ("gpt-4-0314", 17), 80 | ("gpt-4-0613", 17), 81 | ("gpt-4-32k", 17), 82 | ("gpt-4-32k-0314", 17), 83 | ("gpt-4-1106-preview", 17), 84 | ("gpt-4-vision-preview", 17), 85 | ("gpt-4o", 17), 86 | ("azure/gpt-4o", 17), 87 | # ("claude-3-opus-latest", 4), # NOTE: Claude only supports messages without extra inputs 88 | ], 89 | ) 90 | def test_count_message_tokens_with_name(model, expected_output): 91 | """Notice: name 'John' appears""" 92 | 93 | assert count_message_tokens(MESSAGES_WITH_NAME, model) == expected_output 94 | 95 | 96 | def test_count_message_tokens_empty_input(): 97 | """Empty input should raise a KeyError""" 98 | with pytest.raises(KeyError): 99 | count_message_tokens("", "") 100 | 101 | 102 | def test_count_message_tokens_invalid_model(): 103 | """Invalid model should raise a KeyError""" 104 | 105 | with pytest.raises(KeyError): 106 | count_message_tokens(MESSAGES, model="invalid_model") 107 | 108 | 109 | @pytest.mark.parametrize( 110 | "model,expected_output", 111 | [ 112 | ("gpt-3.5-turbo", 4), 113 | ("gpt-3.5-turbo-0301", 4), 114 | ("gpt-3.5-turbo-0613", 4), 115 | ("gpt-3.5-turbo-16k", 4), 116 | ("gpt-3.5-turbo-16k-0613", 4), 117 | ("gpt-3.5-turbo-1106", 4), 118 | ("gpt-3.5-turbo-instruct", 4), 119 | ("gpt-4-0314", 4), 120 | ("gpt-4", 4), 121 | ("gpt-4-32k", 4), 122 | ("gpt-4-32k-0314", 4), 123 | ("gpt-4-0613", 4), 124 | ("gpt-4-1106-preview", 4), 125 | ("gpt-4-vision-preview", 4), 126 | ("text-embedding-ada-002", 4), 127 | ("gpt-4o", 4), 128 | # ("claude-3-opus-latest", 4), # NOTE: Claude only supports messages 129 | ], 130 | ) 131 | def test_count_string_tokens(model, expected_output): 132 | """Test that the string tokens are counted correctly.""" 133 | 134 | # 4 tokens 135 | assert count_string_tokens(STRING, model=model) == expected_output 136 | 137 | # empty string 138 | assert count_string_tokens("", model=model) == 0 139 | 140 | 141 | def test_count_string_invalid_model(): 142 | """Test that the string tokens are counted correctly.""" 143 | 144 | assert count_string_tokens(STRING, model="invalid model") == 4 145 | 146 | 147 | # Costs from https://openai.com/pricing 148 | # https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo 149 | @pytest.mark.parametrize( 150 | "prompt,model,expected_output", 151 | [ 152 | (MESSAGES, "gpt-3.5-turbo", Decimal("0.0000225")), 153 | (MESSAGES, "gpt-3.5-turbo-0301", Decimal("0.0000255")), 154 | (MESSAGES, "gpt-3.5-turbo-0613", Decimal("0.0000225")), 155 | (MESSAGES, "gpt-3.5-turbo-16k", Decimal("0.000045")), 156 | (MESSAGES, "gpt-3.5-turbo-16k-0613", Decimal("0.000045")), 157 | (MESSAGES, "gpt-3.5-turbo-1106", Decimal("0.000015")), 158 | (MESSAGES, "gpt-3.5-turbo-instruct", Decimal("0.0000225")), 159 | (MESSAGES, "gpt-4", Decimal("0.00045")), 160 | (MESSAGES, "gpt-4-0314", Decimal("0.00045")), 161 | (MESSAGES, "gpt-4-32k", Decimal("0.00090")), 162 | (MESSAGES, "gpt-4-32k-0314", Decimal("0.00090")), 163 | (MESSAGES, "gpt-4-0613", Decimal("0.00045")), 164 | (MESSAGES, "gpt-4-1106-preview", Decimal("0.00015")), 165 | (MESSAGES, "gpt-4-vision-preview", Decimal("0.00015")), 166 | (MESSAGES, "gpt-4o", Decimal("0.0000375")), 167 | (MESSAGES, "azure/gpt-4o", Decimal("0.0000375")), 168 | pytest.param(MESSAGES, "claude-3-opus-latest", Decimal("0.000165"), 169 | marks=pytest.mark.skipif( 170 | bool(os.getenv("ANTHROPIC_API_KEY")), 171 | reason="ANTHROPIC_API_KEY environment variable not set" 172 | )), 173 | (STRING, "text-embedding-ada-002", Decimal("0.0000004")), 174 | ], 175 | ) 176 | def test_calculate_prompt_cost(prompt, model, expected_output): 177 | """Test that the cost calculation is correct.""" 178 | 179 | cost = calculate_prompt_cost(prompt, model) 180 | assert cost == expected_output 181 | 182 | 183 | def test_invalid_prompt_format(): 184 | with pytest.raises(TypeError): 185 | calculate_prompt_cost( 186 | {"role": "user", "content": "invalid message type"}, model="gpt-3.5-turbo" 187 | ) 188 | 189 | 190 | @pytest.mark.parametrize( 191 | "prompt,model,expected_output", 192 | [ 193 | (STRING, "gpt-3.5-turbo", Decimal("0.000008")), 194 | (STRING, "gpt-3.5-turbo-0301", Decimal("0.000008")), 195 | (STRING, "gpt-3.5-turbo-0613", Decimal("0.000008")), 196 | (STRING, "gpt-3.5-turbo-16k", Decimal("0.000016")), 197 | (STRING, "gpt-3.5-turbo-16k-0613", Decimal("0.000016")), 198 | (STRING, "gpt-3.5-turbo-1106", Decimal("0.000008")), 199 | (STRING, "gpt-3.5-turbo-instruct", Decimal("0.000008")), 200 | (STRING, "gpt-4", Decimal("0.00024")), 201 | (STRING, "gpt-4-0314", Decimal("0.00024")), 202 | (STRING, "gpt-4-32k", Decimal("0.00048")), 203 | (STRING, "gpt-4-32k-0314", Decimal("0.00048")), 204 | (STRING, "gpt-4-0613", Decimal("0.00024")), 205 | (STRING, "gpt-4-1106-preview", Decimal("0.00012")), 206 | (STRING, "gpt-4-vision-preview", Decimal("0.00012")), 207 | (STRING, "gpt-4o", Decimal("0.00004")), 208 | (STRING, "azure/gpt-4o", Decimal("0.00004")), 209 | # (STRING, "claude-3-opus-latest", Decimal("0.000096")), # NOTE: Claude only supports messages 210 | (STRING, "text-embedding-ada-002", 0), 211 | ], 212 | ) 213 | def test_calculate_completion_cost(prompt, model, expected_output): 214 | """Test that the completion cost calculation is correct.""" 215 | 216 | cost = calculate_completion_cost(prompt, model) 217 | assert cost == expected_output 218 | 219 | 220 | def test_calculate_cost_invalid_model(): 221 | """Invalid model should raise a KeyError""" 222 | 223 | with pytest.raises(KeyError): 224 | calculate_prompt_cost(STRING, model="invalid_model") 225 | 226 | 227 | def test_calculate_invalid_input_types(): 228 | """Invalid input type should raise a KeyError""" 229 | 230 | with pytest.raises(KeyError): 231 | calculate_prompt_cost(STRING, model="invalid_model") 232 | 233 | with pytest.raises(KeyError): 234 | calculate_completion_cost(STRING, model="invalid_model") 235 | 236 | with pytest.raises(KeyError): 237 | # Message objects not allowed, must be list of message objects. 238 | calculate_prompt_cost(MESSAGES[0], model="invalid_model") 239 | 240 | 241 | @pytest.mark.parametrize( 242 | "num_tokens,model,token_type,expected_output", 243 | [ 244 | (10, "gpt-3.5-turbo", "input", Decimal("0.0000150")), # Example values 245 | (5, "gpt-4", "output", Decimal("0.00030")), # Example values 246 | (10, "ai21.j2-mid-v1", "input", Decimal("0.0001250")), # Example values 247 | (100, "gpt-4o", "cached", Decimal("0.000125")), # Cache tokens test 248 | ], 249 | ) 250 | def test_calculate_cost_by_tokens(num_tokens, model, token_type, expected_output): 251 | """Test that the token cost calculation is correct.""" 252 | cost = calculate_cost_by_tokens(num_tokens, model, token_type) 253 | assert cost == expected_output 254 | 255 | 256 | def test_calculate_cached_tokens_cost(): 257 | """Test that cached tokens cost calculation works correctly.""" 258 | # Basic test for cache token cost calculation 259 | model = "gpt-4o" 260 | num_tokens = 1000 261 | token_type = "cached" 262 | 263 | # Get the expected cost from the TOKEN_COSTS dictionary 264 | from tokencost.constants import TOKEN_COSTS 265 | cache_cost_per_token = TOKEN_COSTS[model]["cache_read_input_token_cost"] 266 | expected_cost = Decimal(str(cache_cost_per_token)) * Decimal(num_tokens) 267 | 268 | # Calculate the actual cost 269 | actual_cost = calculate_cost_by_tokens(num_tokens, model, token_type) 270 | 271 | # Assert that the costs match 272 | assert actual_cost == expected_cost 273 | assert actual_cost > 0, "Cache token cost should be greater than zero" 274 | -------------------------------------------------------------------------------- /tokencost.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AgentOps-AI/tokencost/e0c264ec85ed215d6159a5aa56660a0237761771/tokencost.png -------------------------------------------------------------------------------- /tokencost/__init__.py: -------------------------------------------------------------------------------- 1 | from .costs import ( 2 | count_message_tokens, 3 | count_string_tokens, 4 | calculate_completion_cost, 5 | calculate_prompt_cost, 6 | calculate_all_costs_and_tokens, 7 | calculate_cost_by_tokens, 8 | ) 9 | from .constants import TOKEN_COSTS_STATIC, TOKEN_COSTS, update_token_costs, refresh_prices 10 | -------------------------------------------------------------------------------- /tokencost/constants.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import aiohttp 4 | import asyncio 5 | import logging 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | """ 10 | Prompt (aka context) tokens are based on number of words + other chars (eg spaces and punctuation) in input. 11 | Completion tokens are similarly based on how long chatGPT's response is. 12 | Prompt tokens + completion tokens = total tokens. 13 | The max total limit is typically 1 more than the prompt token limit, so there's space for at least one completion token. 14 | 15 | You can use ChatGPT's webapp (which uses their tiktoken repo) to see how many tokens your phrase is: 16 | https://platform.openai.com/tokenizer 17 | 18 | Note: When asking follow-up questions, everything above and including your follow-up question 19 | is considered a prompt (for the purpose of context) and will thus cost prompt tokens. 20 | """ 21 | 22 | # How to read TOKEN_COSTS: 23 | # Each prompt token costs __ USD per token. 24 | # Each completion token costs __ USD per token. 25 | # Max prompt limit of each model is __ tokens. 26 | 27 | PRICES_URL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" 28 | 29 | 30 | async def fetch_costs(): 31 | """Fetch the latest token costs from the LiteLLM cost tracker asynchronously. 32 | Returns: 33 | dict: The token costs for each model. 34 | Raises: 35 | Exception: If the request fails. 36 | """ 37 | async with aiohttp.ClientSession(trust_env=True) as session: 38 | async with session.get(PRICES_URL) as response: 39 | if response.status == 200: 40 | return await response.json(content_type=None) 41 | else: 42 | raise Exception( 43 | f"Failed to fetch token costs, status code: {response.status}" 44 | ) 45 | 46 | 47 | async def update_token_costs(): 48 | """Update the TOKEN_COSTS dictionary with the latest costs from the LiteLLM cost tracker asynchronously.""" 49 | global TOKEN_COSTS 50 | try: 51 | fetched_costs = await fetch_costs() 52 | # Safely remove 'sample_spec' if it exists 53 | TOKEN_COSTS.update(fetched_costs) 54 | TOKEN_COSTS.pop("sample_spec", None) 55 | return TOKEN_COSTS 56 | except Exception as e: 57 | logger.error(f"Failed to update TOKEN_COSTS: {e}") 58 | raise 59 | 60 | 61 | def refresh_prices(write_file=True): 62 | """Synchronous wrapper for update_token_costs that optionally writes to model_prices.json.""" 63 | try: 64 | # Run the async function in a new event loop 65 | updated_costs = asyncio.run(update_token_costs()) 66 | 67 | # Write to file if requested 68 | if write_file: 69 | file_path = os.path.join(os.path.dirname(__file__), "model_prices.json") 70 | with open(file_path, "w") as f: 71 | json.dump(TOKEN_COSTS, f, indent=4) 72 | logger.info(f"Updated prices written to {file_path}") 73 | 74 | return updated_costs 75 | except Exception as e: 76 | logger.error(f"Failed to refresh prices: {e}") 77 | # Return the static prices as fallback 78 | return TOKEN_COSTS 79 | 80 | 81 | with open(os.path.join(os.path.dirname(__file__), "model_prices.json"), "r") as f: 82 | TOKEN_COSTS_STATIC = json.load(f) 83 | 84 | 85 | # Set initial TOKEN_COSTS to the static values 86 | TOKEN_COSTS = TOKEN_COSTS_STATIC.copy() 87 | 88 | # Only run in a non-async context 89 | if __name__ == "__main__": 90 | try: 91 | asyncio.run(update_token_costs()) 92 | print("Token costs updated successfully") 93 | except Exception: 94 | logger.error("Failed to update token costs. Using static costs.") 95 | -------------------------------------------------------------------------------- /tokencost/costs.py: -------------------------------------------------------------------------------- 1 | """ 2 | Costs dictionary and utility tool for counting tokens 3 | """ 4 | 5 | import os 6 | import tiktoken 7 | import anthropic 8 | from typing import Union, List, Dict, Literal 9 | from .constants import TOKEN_COSTS 10 | from decimal import Decimal 11 | import logging 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | # Note: cl100k is the openai base tokenizer. Nothing to do with Claude. Tiktoken doesn't have claude yet. 16 | # https://github.com/anthropics/anthropic-tokenizer-typescript/blob/main/index.ts 17 | 18 | 19 | TokenType = Literal["input", "output", "cached"] 20 | 21 | 22 | def _get_field_from_token_type(token_type: TokenType) -> str: 23 | """ 24 | Get the field name from the token type. 25 | 26 | Args: 27 | token_type (TokenType): The token type. 28 | 29 | Returns: 30 | str: The field name to use for the token cost data in the TOKEN_COSTS dictionary. 31 | """ 32 | lookups = { 33 | "input": "input_cost_per_token", 34 | "output": "output_cost_per_token", 35 | "cached": "cache_read_input_token_cost", 36 | } 37 | 38 | try: 39 | return lookups[token_type] 40 | except KeyError: 41 | raise ValueError(f"Invalid token type: {token_type}.") 42 | 43 | 44 | def get_anthropic_token_count(messages: List[Dict[str, str]], model: str) -> int: 45 | if not any( 46 | supported_model in model 47 | for supported_model in [ 48 | "claude-3-7-sonnet", 49 | "claude-3-5-sonnet", 50 | "claude-3-5-haiku", 51 | "claude-3-haiku", 52 | "claude-3-opus", 53 | ] 54 | ): 55 | raise ValueError( 56 | f"{model} is not supported in token counting (beta) API. Use the `usage` property in the response for exact counts." 57 | ) 58 | try: 59 | return ( 60 | anthropic.Anthropic() 61 | .beta.messages.count_tokens( 62 | model=model, 63 | messages=messages, 64 | ) 65 | .input_tokens 66 | ) 67 | except TypeError as e: 68 | raise e 69 | except Exception as e: 70 | raise e 71 | 72 | 73 | def strip_ft_model_name(model: str) -> str: 74 | """ 75 | Finetuned models format: ft:gpt-3.5-turbo:my-org:custom_suffix:id 76 | We only need the base model name to get cost info. 77 | """ 78 | if model.startswith("ft:gpt-3.5-turbo"): 79 | model = "ft:gpt-3.5-turbo" 80 | return model 81 | 82 | 83 | def count_message_tokens(messages: List[Dict[str, str]], model: str) -> int: 84 | """ 85 | Return the total number of tokens in a prompt's messages. 86 | Args: 87 | messages (List[Dict[str, str]]): Message format for prompt requests. e.g.: 88 | [{ "role": "user", "content": "Hello world"}, 89 | { "role": "assistant", "content": "How may I assist you today?"}] 90 | model (str): Name of LLM to choose encoding for. 91 | Returns: 92 | Total number of tokens in message. 93 | """ 94 | model = model.lower() 95 | model = strip_ft_model_name(model) 96 | 97 | # Anthropic token counting requires a valid API key 98 | if "claude-" in model: 99 | logger.warning( 100 | "Warning: Anthropic token counting API is currently in beta. Please expect differences in costs!" 101 | ) 102 | return get_anthropic_token_count(messages, model) 103 | 104 | try: 105 | encoding = tiktoken.encoding_for_model(model) 106 | except KeyError: 107 | logger.warning("Model not found. Using cl100k_base encoding.") 108 | encoding = tiktoken.get_encoding("cl100k_base") 109 | if model in { 110 | "gpt-3.5-turbo-0613", 111 | "gpt-3.5-turbo-16k-0613", 112 | "gpt-4-0314", 113 | "gpt-4-32k-0314", 114 | "gpt-4-0613", 115 | "gpt-4-32k-0613", 116 | "gpt-4-turbo", 117 | "gpt-4-turbo-2024-04-09", 118 | "gpt-4o", 119 | "gpt-4o-2024-05-13", 120 | } or model.startswith("o"): 121 | tokens_per_message = 3 122 | tokens_per_name = 1 123 | elif model == "gpt-3.5-turbo-0301": 124 | # every message follows <|start|>{role/name}\n{content}<|end|>\n 125 | tokens_per_message = 4 126 | tokens_per_name = -1 # if there's a name, the role is omitted 127 | elif "gpt-3.5-turbo" in model: 128 | logger.warning( 129 | "gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613." 130 | ) 131 | return count_message_tokens(messages, model="gpt-3.5-turbo-0613") 132 | elif "gpt-4o" in model: 133 | logger.warning( 134 | "Warning: gpt-4o may update over time. Returning num tokens assuming gpt-4o-2024-05-13." 135 | ) 136 | return count_message_tokens(messages, model="gpt-4o-2024-05-13") 137 | elif "gpt-4" in model: 138 | logger.warning( 139 | "gpt-4 may update over time. Returning num tokens assuming gpt-4-0613." 140 | ) 141 | return count_message_tokens(messages, model="gpt-4-0613") 142 | else: 143 | raise KeyError( 144 | f"""num_tokens_from_messages() is not implemented for model {model}. 145 | See https://github.com/openai/openai-python/blob/main/chatml.md for how messages are converted to tokens.""" 146 | ) 147 | num_tokens = 0 148 | for message in messages: 149 | num_tokens += tokens_per_message 150 | for key, value in message.items(): 151 | num_tokens += len(encoding.encode(value)) 152 | if key == "name": 153 | num_tokens += tokens_per_name 154 | num_tokens += 3 # every reply is primed with <|start|>assistant<|message|> 155 | return num_tokens 156 | 157 | 158 | def count_string_tokens(prompt: str, model: str) -> int: 159 | """ 160 | Returns the number of tokens in a (prompt or completion) text string. 161 | 162 | Args: 163 | prompt (str): The text string 164 | model_name (str): The name of the encoding to use. (e.g., "gpt-3.5-turbo") 165 | 166 | Returns: 167 | int: The number of tokens in the text string. 168 | """ 169 | model = model.lower() 170 | 171 | if "/" in model: 172 | model = model.split("/")[-1] 173 | 174 | if "claude-" in model: 175 | raise ValueError( 176 | "Warning: Anthropic does not support this method. Please use the `count_message_tokens` function for the exact counts." 177 | ) 178 | 179 | try: 180 | encoding = tiktoken.encoding_for_model(model) 181 | except KeyError: 182 | logger.warning("Warning: model not found. Using cl100k_base encoding.") 183 | encoding = tiktoken.get_encoding("cl100k_base") 184 | 185 | return len(encoding.encode(prompt)) 186 | 187 | 188 | def calculate_cost_by_tokens(num_tokens: int, model: str, token_type: TokenType) -> Decimal: 189 | """ 190 | Calculate the cost based on the number of tokens and the model. 191 | 192 | Args: 193 | num_tokens (int): The number of tokens. 194 | model (str): The model name. 195 | token_type (str): Type of token ('input' or 'output'). 196 | 197 | Returns: 198 | Decimal: The calculated cost in USD. 199 | """ 200 | model = model.lower() 201 | if model not in TOKEN_COSTS: 202 | raise KeyError( 203 | f"""Model {model} is not implemented. 204 | Double-check your spelling, or submit an issue/PR""" 205 | ) 206 | 207 | try: 208 | token_key = _get_field_from_token_type(token_type) 209 | cost_per_token = TOKEN_COSTS[model][token_key] 210 | except KeyError: 211 | raise KeyError(f"Model {model} does not have cost data for `{token_type}` tokens.") 212 | 213 | return Decimal(str(cost_per_token)) * Decimal(num_tokens) 214 | 215 | 216 | def calculate_prompt_cost(prompt: Union[List[dict], str], model: str) -> Decimal: 217 | """ 218 | Calculate the prompt's cost in USD. 219 | 220 | Args: 221 | prompt (Union[List[dict], str]): List of message objects or single string prompt. 222 | model (str): The model name. 223 | 224 | Returns: 225 | Decimal: The calculated cost in USD. 226 | 227 | e.g.: 228 | >>> prompt = [{ "role": "user", "content": "Hello world"}, 229 | { "role": "assistant", "content": "How may I assist you today?"}] 230 | >>>calculate_prompt_cost(prompt, "gpt-3.5-turbo") 231 | Decimal('0.0000300') 232 | # or 233 | >>> prompt = "Hello world" 234 | >>> calculate_prompt_cost(prompt, "gpt-3.5-turbo") 235 | Decimal('0.0000030') 236 | """ 237 | model = model.lower() 238 | model = strip_ft_model_name(model) 239 | if model not in TOKEN_COSTS: 240 | raise KeyError( 241 | f"""Model {model} is not implemented. 242 | Double-check your spelling, or submit an issue/PR""" 243 | ) 244 | if not isinstance(prompt, (list, str)): 245 | raise TypeError( 246 | f"Prompt must be either a string or list of message objects but found {type(prompt)} instead." 247 | ) 248 | prompt_tokens = ( 249 | count_string_tokens(prompt, model) 250 | if isinstance(prompt, str) and "claude-" not in model 251 | else count_message_tokens(prompt, model) 252 | ) 253 | 254 | return calculate_cost_by_tokens(prompt_tokens, model, "input") 255 | 256 | 257 | def calculate_completion_cost(completion: str, model: str) -> Decimal: 258 | """ 259 | Calculate the prompt's cost in USD. 260 | 261 | Args: 262 | completion (str): Completion string. 263 | model (str): The model name. 264 | 265 | Returns: 266 | Decimal: The calculated cost in USD. 267 | 268 | e.g.: 269 | >>> completion = "How may I assist you today?" 270 | >>> calculate_completion_cost(completion, "gpt-3.5-turbo") 271 | Decimal('0.000014') 272 | """ 273 | model = strip_ft_model_name(model) 274 | if model not in TOKEN_COSTS: 275 | raise KeyError( 276 | f"""Model {model} is not implemented. 277 | Double-check your spelling, or submit an issue/PR""" 278 | ) 279 | 280 | if not isinstance(completion, str): 281 | raise TypeError( 282 | f"Prompt must be a string but found {type(completion)} instead." 283 | ) 284 | 285 | if "claude-" in model: 286 | completion_list = [{"role": "assistant", "content": completion}] 287 | # Anthropic appends some 13 additional tokens to the actual completion tokens 288 | completion_tokens = count_message_tokens(completion_list, model) - 13 289 | else: 290 | completion_tokens = count_string_tokens(completion, model) 291 | 292 | return calculate_cost_by_tokens(completion_tokens, model, "output") 293 | 294 | 295 | def calculate_all_costs_and_tokens( 296 | prompt: Union[List[dict], str], completion: str, model: str 297 | ) -> dict: 298 | """ 299 | Calculate the prompt and completion costs and tokens in USD. 300 | 301 | Args: 302 | prompt (Union[List[dict], str]): List of message objects or single string prompt. 303 | completion (str): Completion string. 304 | model (str): The model name. 305 | 306 | Returns: 307 | dict: The calculated cost and tokens in USD. 308 | 309 | e.g.: 310 | >>> prompt = "Hello world" 311 | >>> completion = "How may I assist you today?" 312 | >>> calculate_all_costs_and_tokens(prompt, completion, "gpt-3.5-turbo") 313 | {'prompt_cost': Decimal('0.0000030'), 'prompt_tokens': 2, 'completion_cost': Decimal('0.000014'), 'completion_tokens': 7} 314 | """ 315 | prompt_cost = calculate_prompt_cost(prompt, model) 316 | completion_cost = calculate_completion_cost(completion, model) 317 | prompt_tokens = ( 318 | count_string_tokens(prompt, model) 319 | if isinstance(prompt, str) and "claude-" not in model 320 | else count_message_tokens(prompt, model) 321 | ) 322 | 323 | if "claude-" in model: 324 | logger.warning("Warning: Token counting is estimated for ") 325 | completion_list = [{"role": "assistant", "content": completion}] 326 | # Anthropic appends some 13 additional tokens to the actual completion tokens 327 | completion_tokens = count_message_tokens(completion_list, model) - 13 328 | else: 329 | completion_tokens = count_string_tokens(completion, model) 330 | 331 | return { 332 | "prompt_cost": prompt_cost, 333 | "prompt_tokens": prompt_tokens, 334 | "completion_cost": completion_cost, 335 | "completion_tokens": completion_tokens, 336 | } 337 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py3, flake8 3 | isolated_build = true 4 | 5 | [testenv] 6 | deps = 7 | pytest 8 | coverage 9 | commands = 10 | coverage run --source tokencost -m pytest {posargs} 11 | coverage report -m 12 | 13 | [testenv:flake8] 14 | deps = flake8 15 | commands = flake8 tokencost/ 16 | 17 | [flake8] 18 | max-line-length = 120 19 | per-file-ignores = 20 | tokencost/__init__.py: F401 -------------------------------------------------------------------------------- /update_prices.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import tokencost 3 | from decimal import Decimal 4 | import json 5 | import re 6 | 7 | # Update model_prices.json with the latest costs from the LiteLLM cost tracker 8 | print("Fetching latest prices...") 9 | tokencost.refresh_prices(write_file=False) 10 | 11 | 12 | def diff_dicts(dict1, dict2): 13 | diff_keys = dict1.keys() ^ dict2.keys() 14 | differences = {k: (dict1.get(k), dict2.get(k)) for k in diff_keys} 15 | differences.update( 16 | {k: (dict1[k], dict2[k]) for k in dict1 if k in dict2 and dict1[k] != dict2[k]} 17 | ) 18 | 19 | if differences: 20 | print("Differences found:") 21 | for key, (val1, val2) in differences.items(): 22 | print(f"{key}: {val1} != {val2}") 23 | else: 24 | print("No differences found.") 25 | 26 | return bool(differences) 27 | 28 | 29 | # Load the current file for comparison 30 | with open("tokencost/model_prices.json", "r") as f: 31 | model_prices = json.load(f) 32 | 33 | # Compare the refreshed TOKEN_COSTS with the file 34 | if diff_dicts(model_prices, tokencost.TOKEN_COSTS): 35 | print("Updating model_prices.json") 36 | with open("tokencost/model_prices.json", "w") as f: 37 | json.dump(tokencost.TOKEN_COSTS, f, indent=4) 38 | print("File updated successfully") 39 | else: 40 | print("File is already up to date") 41 | 42 | # Load the data 43 | df = pd.DataFrame(tokencost.TOKEN_COSTS).T 44 | df.loc[df.index[1:], "max_input_tokens"] = ( 45 | df["max_input_tokens"].iloc[1:].apply(lambda x: "{:,.0f}".format(x)) 46 | ) 47 | df.loc[df.index[1:], "max_tokens"] = ( 48 | df["max_tokens"].iloc[1:].apply(lambda x: "{:,.0f}".format(x)) 49 | ) 50 | 51 | 52 | # Updated function to format the cost or handle NaN 53 | def format_cost(x): 54 | if pd.isna(x): 55 | return "--" 56 | else: 57 | price_per_million = Decimal(str(x)) * Decimal(str(1_000_000)) 58 | normalized = price_per_million.normalize() 59 | formatted_price = "{:.2f}".format(normalized) 60 | 61 | formatted_price = ( 62 | formatted_price.rstrip("0").rstrip(".") 63 | if "." in formatted_price 64 | else formatted_price + ".00" 65 | ) 66 | 67 | return f"${formatted_price}" 68 | 69 | 70 | # Apply the formatting function using DataFrame.apply and lambda 71 | df[["input_cost_per_token", "output_cost_per_token"]] = df[ 72 | ["input_cost_per_token", "output_cost_per_token"] 73 | ].apply(lambda x: x.map(format_cost)) 74 | 75 | 76 | column_mapping = { 77 | "input_cost_per_token": "Prompt Cost (USD) per 1M tokens", 78 | "output_cost_per_token": "Completion Cost (USD) per 1M tokens", 79 | "max_input_tokens": "Max Prompt Tokens", 80 | "max_output_tokens": "Max Output Tokens", 81 | "model_name": "Model Name", 82 | } 83 | 84 | # Assuming the keys of the JSON data represent the model names and have been set as the index 85 | df["Model Name"] = df.index 86 | 87 | # Apply the column renaming 88 | df.rename(columns=column_mapping, inplace=True) 89 | 90 | # Generate the markdown table 91 | table_md = df[ 92 | [ 93 | "Model Name", 94 | "Prompt Cost (USD) per 1M tokens", 95 | "Completion Cost (USD) per 1M tokens", 96 | "Max Prompt Tokens", 97 | "Max Output Tokens", 98 | ] 99 | ].to_markdown(index=False) 100 | 101 | # Write the markdown table to pricing_table.md for reference 102 | with open("pricing_table.md", "w") as f: 103 | f.write(table_md) 104 | 105 | # Read the README.md file 106 | with open("README.md", "r") as f: 107 | readme_content = f.read() 108 | 109 | # Find and replace just the table in the README, preserving the header text 110 | # The regex pattern matches a markdown table starting with the "Model Name" header 111 | # and ending before the next markdown header. DOTALL mode is enabled to allow 112 | # the `.` character to match newline characters. 113 | table_pattern = r"(?s)\| Model Name.*?\n\n(?=#)" 114 | table_replacement = table_md 115 | 116 | updated_readme = re.sub(table_pattern, table_replacement, readme_content, flags=re.DOTALL) 117 | 118 | # Write the updated README 119 | with open("README.md", "w") as f: 120 | f.write(updated_readme) 121 | --------------------------------------------------------------------------------