├── .github └── workflows │ └── python-package.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── analysis ├── __init__.py ├── demo.ipynb ├── plotting.py └── utils.py ├── openvalidators ├── __init__.py ├── config.py ├── dataset.py ├── dendrite.py ├── event.py ├── forward.py ├── gating.py ├── misc.py ├── mock.py ├── neuron.py ├── prompts.py ├── reward │ ├── __init__.py │ ├── blacklist.py │ ├── config.py │ ├── dahoas.py │ ├── diversity.py │ ├── dpo.py │ ├── nsfw.py │ ├── open_assistant.py │ ├── prompt.py │ ├── reciprocate.py │ ├── relevance.py │ ├── reward.py │ └── task_validator.py ├── run.py ├── utils.py └── weights.py ├── pytest.ini ├── requirements.txt ├── run.sh ├── scratch.ipynb ├── scripts ├── Makefile ├── README.md ├── blacklist_phrases.txt ├── data_collector.py ├── data_formatter.py └── release │ ├── README.md │ ├── add_notes_changelog.sh │ ├── github_release.sh │ ├── github_utils.sh │ ├── release.sh │ ├── utils.sh │ └── versioning.sh ├── setup.py └── tests ├── __init__.py ├── helpers └── __init__.py ├── reward ├── __init__.py └── test_task_validator.py ├── test_dataset.py ├── test_dendrite.py ├── test_event.py ├── test_utils.py └── test_weights.py /.github/workflows/python-package.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python 3 | 4 | name: Python package 5 | 6 | on: push 7 | 8 | jobs: 9 | build: 10 | 11 | runs-on: ubuntu-latest 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | python-version: ["3.8", "3.9", "3.10"] 16 | 17 | steps: 18 | - uses: actions/checkout@v3 19 | - name: Set up Python ${{ matrix.python-version }} 20 | uses: actions/setup-python@v3 21 | with: 22 | python-version: ${{ matrix.python-version }} 23 | - name: Install dependencies 24 | run: | 25 | python -m pip install --upgrade pip 26 | python -m pip install flake8 pytest 27 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 28 | - name: Lint with flake8 29 | run: | 30 | # stop the build if there are Python syntax errors or undefined names 31 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 32 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 33 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 34 | - name: Test with pytest 35 | run: | 36 | pytest 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Local analyses 2 | analysis/*csv 3 | analysis/*pkl 4 | analysis/*png 5 | analysis/*html 6 | 7 | # Byte-compiled / optimized / DLL files 8 | __pycache__/ 9 | *.py[cod] 10 | *$py.class 11 | 12 | # C extensions 13 | *.so 14 | 15 | # Distribution / packaging 16 | .Python 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | share/python-wheels/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | MANIFEST 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .nox/ 49 | .coverage 50 | .coverage.* 51 | .cache 52 | nosetests.xml 53 | coverage.xml 54 | *.cover 55 | *.py,cover 56 | .hypothesis/ 57 | .pytest_cache/ 58 | cover/ 59 | 60 | # Translations 61 | *.mo 62 | *.pot 63 | 64 | # Django stuff: 65 | *.log 66 | local_settings.py 67 | db.sqlite3 68 | db.sqlite3-journal 69 | 70 | # Flask stuff: 71 | instance/ 72 | .webassets-cache 73 | 74 | # Scrapy stuff: 75 | .scrapy 76 | 77 | # Sphinx documentation 78 | docs/_build/ 79 | 80 | # PyBuilder 81 | .pybuilder/ 82 | target/ 83 | 84 | # Jupyter Notebook 85 | .ipynb_checkpoints 86 | 87 | # IPython 88 | profile_default/ 89 | ipython_config.py 90 | 91 | # pyenv 92 | # For a library or package, you might want to ignore these files since the code is 93 | # intended to run in multiple environments; otherwise, check them in: 94 | # .python-version 95 | 96 | # pipenv 97 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 98 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 99 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 100 | # install all needed dependencies. 101 | #Pipfile.lock 102 | 103 | # poetry 104 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 105 | # This is especially recommended for binary packages to ensure reproducibility, and is more 106 | # commonly ignored for libraries. 107 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 108 | #poetry.lock 109 | 110 | # pdm 111 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 112 | #pdm.lock 113 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 114 | # in version control. 115 | # https://pdm.fming.dev/#use-with-ide 116 | .pdm.toml 117 | 118 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 119 | __pypackages__/ 120 | 121 | # Celery stuff 122 | celerybeat-schedule 123 | celerybeat.pid 124 | 125 | # SageMath parsed files 126 | *.sage.py 127 | 128 | # Environments 129 | .env 130 | .venv 131 | env/ 132 | venv/ 133 | ENV/ 134 | env.bak/ 135 | venv.bak/ 136 | 137 | # Spyder project settings 138 | .spyderproject 139 | .spyproject 140 | 141 | # Rope project settings 142 | .ropeproject 143 | 144 | # mkdocs documentation 145 | /site 146 | 147 | # mypy 148 | .mypy_cache/ 149 | .dmypy.json 150 | dmypy.json 151 | 152 | # Pyre type checker 153 | .pyre/ 154 | 155 | # pytype static type analyzer 156 | .pytype/ 157 | 158 | # Cython debug symbols 159 | cython_debug/ 160 | 161 | # PyCharm 162 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 163 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 164 | # and can be added to the global gitignore or merged into this file. For a more nuclear 165 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 166 | .idea/ 167 | 168 | app.config.js 169 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | files: '.py' 2 | repos: 3 | - repo: https://github.com/kynan/nbstripout 4 | rev: 0.6.0 5 | hooks: 6 | - id: nbstripout 7 | - repo: https://github.com/pre-commit/pre-commit-hooks 8 | rev: v2.3.0 9 | hooks: 10 | - id: check-yaml 11 | - id: end-of-file-fixer 12 | - id: trailing-whitespace 13 | - repo: https://github.com/psf/black 14 | rev: 22.3.0 15 | hooks: 16 | - id: black 17 | args: [-l 127] 18 | - repo: https://github.com/PyCQA/flake8 19 | rev: 5.0.4 20 | hooks: 21 | - id: flake8 22 | args: [--count, --select=E9, --select=F63, --select=F7, --select=F82, --show-source, --statistics] -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 1.2.0 / 2023-08-28 4 | ### What's changed 5 | - Adds Direct Optimization (DPO) style rewards by @opentaco on #99 6 | - Changes print format on exception catch by @camfairchild on #135 7 | - Brings back netuid and wandb to logged config by @p-ferreira on #137 8 | - Adds DPO penalty update by @Eugene-hu on #138 9 | - Adds original reward output to wandb logs by @isabella618033 on #139 10 | - Reweights reward models by @Eugene-hu on #140 11 | - Update stale documentation by @steffencruz on #129 12 | 13 | 14 | 15 | **Full Changelog**: https://github.com/opentensor/validators/compare/v1.1.7...v1.2.0 16 | 17 | ## 1.1.8 / 2023-08-12 18 | ### What's Changed 19 | - Make sure to serve axon first by @camfairchild in 14921d35c 20 | - Adds scripts for releases on github by @camfairchild in #128 21 | - Wandb config log changes @isabella618033 in #132 22 | 23 | ## 1.1.7 / 2023-08-11 24 | ### What’s Changed 25 | - Hotfix cutoff limit by @Eugene-hu in #126 26 | 27 | ## 1.1.6 / 2023-08-10 28 | ### What’s Changed 29 | - Diversity regularization by @isabella618033 in https://github.com/opentensor/validators/pull/124 30 | 31 | ## 1.1.5 / 2023-08-08 32 | ### What’s Changed 33 | - Adds new keywords for the task validator by @p-ferreira in #119 34 | - Save historic embeddings on disk by @opentaco in #121 35 | - Updates relevance mechanism by @Eugene-hu in #122 36 | 37 | ## 1.1.4 / 2023-08-07 38 | - HOTFIX: create and serve the axon at startup by @robertalanm in #120 39 | 40 | 41 | ## 1.1.3 / 2023-08-02 42 | - Adds subtensor to metagraph sync by @camfairchild in #79 43 | - Fix wandb weights format logging by @p-ferreira in #88 44 | - Adds netuid tag to wandb runs by @p-ferreira in #95 45 | - Implements GPU cleaning for optmization by @Eugene-hu in #96 46 | - Adds compatibility with bittensor 5.3.3 by @camfairchild in #107 47 | - Adds historic diversity component by @isabella618033 in #111 48 | - Improvements on diveristy model by @isabella618033 and @Eugene-hu in #111 49 | - Prompt improvements by @mrseeker in #110 and @p-ferreira in #112 50 | - Adds Task Validator Filter to reward pipeline by @p-ferreira in #112 51 | - Fix for empty data retrieval from datasets by @p-ferreira in #113 52 | - Deprecates pip usage by @p-ferreira in #114 53 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright © 2023 Yuma Rao 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | the Software. 11 | 12 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | # **Open Validators** 4 | [![Discord Chat](https://img.shields.io/discord/308323056592486420.svg)](https://discord.gg/bittensor) 5 | [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) 6 | 7 | --- 8 | 9 |
10 | 11 | ## ⚠️ **Deprecated**: This project is no longer maintained. We recommend using [text-prompting repository](https://github.com/opentensor/text-prompting) instead. 12 | 13 | This repository contains Bittensor Validators designed by the OpenTensor Foundation team for the community. 14 | It offers several functionalities, such as: 15 | 16 | - Building and running Bittensor validators 17 | - Real-time analysis of validator performance integrated with wandb 18 | - Offline analysis of data generated from the network 19 | - Creation of datasets using network data for training miners 20 | 21 | The main goal of this repository is to facilitate the interaction with the Bittensor network by providing a set of 22 | open-source validators to the community. The current validator implementation queries the network for responses and 23 | evaluations using carefully crafted prompts using CoT, that are later evaluated by a pipeline of reward functions, including diversity, relevance, rlhf, among others. 24 | 25 | Additionally, the repository provides an analysis and data toolkit that allows users to analyze the data generated from 26 | the validator's interaction with the network. By default, the validator collects various data points, such as question 27 | responses, evaluations, rewards and scorings by UID, and model performance data. This data is then sent to wandb, 28 | making it publicly accessible to the community. 29 | 30 | The toolkit also includes scripts to analyze and extract data from specific validator runs or multiple runs, simplifying 31 | the creation of valuable datasets for the community's miners. 32 | 33 | To learn more about the Bittensor validation process, check out this [documentation](https://tensor-wiki.vercel.app/validating/validating). 34 | 35 | # Running 36 | 37 | These validators are designed to run and update themselves automatically. To run a validator, follow these steps: 38 | 39 | 1. Install this repository, you can do so by following the steps outlined in [the installation section](#install). 40 | 2. Install [Weights and Biases](https://docs.wandb.ai/quickstart) and run `wandb login` within this repository. This will initialize Weights and Biases, enabling you to view KPIs and Metrics on your validator. (Strongly recommended to help the network improve from data sharing) 41 | 3. Install [PM2](https://pm2.io/docs/runtime/guide/installation/) and the [`jq` package](https://jqlang.github.io/jq/) on your system. 42 | **On Linux**: 43 | ```bash 44 | sudo apt update && sudo apt install jq && sudo apt install npm && sudo npm install pm2 -g && pm2 update 45 | ``` 46 | **On Mac OS** 47 | ```bash 48 | brew update && brew install jq && brew install npm && sudo npm install pm2 -g && pm2 update 49 | ``` 50 | 4. Run the `run.sh` script which will handle running your validator and pulling the latest updates as they are issued. 51 | ```bash 52 | pm2 start run.sh --name openvalidators_autoupdate -- --wallet.name --wallet.hotkey 53 | ``` 54 | 55 | This will run **two** PM2 process: one for the validator which is called `auto_run_validator` by default (you can change this in `run.sh`), and one for the run.sh script (in step 4, we named it `validator_maintainer`). The script will check for updates every 30 minutes, if there is an update then it will pull it, install it, restart `auto_run_validator` and then restart itself. 56 | 57 | # Usage 58 | There are currently four main avenues for engaging with this repository: 59 | 60 | 1. [Validators](#Validators): 61 | - Designed for TAO holders who aim to build or run validators developed by the foundation. 62 | 63 | 2. [Real-time monitoring with wandb integration](#Real-time-monitoring-with-wandb-integration): 64 | - Allows users to analyze the performance of various validators runs in real-time using wandb. 65 | 66 | 3. [Network analysis](#Network-analysis) 67 | - Caters to individuals, researchers, and data scientists interested in analyzing the data generated from the validators' interaction with the Bittensor network. 68 | 69 | 4. [Dataset creation](#Dataset-creation) 70 | - Serves individuals, researchers, and developers who seek to create datasets for the community's miners. 71 | 72 | # Install 73 | From source: 74 | ```bash 75 | $ git clone https://github.com/opentensor/validators.git 76 | $ pip3 install -e openvalidators/ 77 | ``` 78 | 79 | You can test the installation by running the following command: 80 | ```bash 81 | $ python3 validators/openvalidators/neuron.py --help 82 | ``` 83 | 84 | # Validators 85 | Participation in Network Validation is available to TAO holders. The validation mechanism utilizes a dual proof-of-stake and proof-of-work system known as *Yuma Consensus*, which you can learn more about [here](https://tensor-wiki.vercel.app/validating/validating). To start validating, you will need to have a Bittensor wallet with a sufficient amount of TAO tokens staked. 86 | 87 | Once you have your wallet ready for validation, you can start the foundation validator by running the following command: 88 | ```bash 89 | $ python3 validators/openvalidators/neuron.py --wallet.name --wallet.hotkey 90 | ``` 91 | 92 | # Real-time monitoring with wandb integration 93 | By default, the validator sends data to wandb, allowing users to monitor running validators and access key metrics in real time, such as: 94 | - Gating model loss 95 | - Hardware usage 96 | - Forward pass time 97 | - Block duration 98 | 99 | All the data sent to wandb is publicly available to the community at the following [link](https://wandb.ai/opentensor-dev/openvalidators). 100 | 101 | You don't need to have a wandb account to access the data or to generate a new run, 102 | but bear in mind that 103 | [data generated by anonymous users will be deleted after 7 days](https://docs.wandb.ai/guides/app/features/anon#:~:text=If%20there's%20no%20account%2C%20we,be%20available%20for%207%20days) 104 | as default wandb policy. 105 | 106 | # Network analysis 107 | This repository provides a set of tools to analyze the data generated by the validators, including: 108 | - Completions 109 | - Rewards 110 | - Weights 111 | - [Prompt scoring](#Prompt-based-scoring) 112 | 113 | A basic tutorial for downloading and analyzing wandb data can be found in [analysis](./analysis/demo.ipynb). 114 | 115 | # Dataset creation 116 | For the individuals who are eager to create datasets tailored specifically for the community's miners. 117 | With convenient scripts available in the [scripts](./scripts) folder, you can effortlessly download data from specific or multiple runs 118 | of wandb, empowering you to curate comprehensive and valuable datasets that align with your mining objectives. 119 | Check the [README of the data collector](./scripts/README.md) for more information. 120 | 121 | ---- 122 | ## Experimental Features 123 | 124 | ## Sentence Embedding Gating Model 125 | Another cornerstone of the validator functionality is the use of a mixture of experts (MoE) model, which we call the gating model, to enable queries to be efficiently routed to the best-suited miners. **This incentivizes miners to become specialists, which in turn improves response quality**. It also reduces latency and addresses bandwidth issues in the network. 126 | We are working on a new and improved gating model, based on sentence embeddings, which is expected to be a more powerful and robust router for queries. By default it is disabled, but can be enabled with the flags 127 | 128 | ```--neuron.use_custom_gating_model --gating.model_name sentence-transformers/all-distilroberta-v1``` 129 | 130 | ## CUDA device placement 131 | If you desire to place your validator on a specific GPU, it is recommended to prepend the command you are using to start and run your validator with `CUDA_VISIBLE_DEVICES`. 132 | 133 | For running with pm2: 134 | ```bash 135 | $ CUDA_VISIBLE_DEVICES= pm2 start run.sh --name openvalidators_autoupdate -- --wallet.name --wallet.hotkey 136 | ``` 137 | 138 | For runing `neuron.py` directly: 139 | ```bash 140 | $ CUDA_VISIBLE_DEVICES= python3 validators/openvalidators/neuron.py --wallet.name --wallet.hotkey 141 | ``` 142 | # License 143 | 144 | The MIT License (MIT) Copyright © 2023 Yuma Rao 145 | 146 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 147 | documentation files (the “Software”), to deal in the Software without restriction, including without limitation the 148 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit 149 | persons to whom the Software is furnished to do so, subject to the following conditions: 150 | 151 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 152 | 153 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 154 | WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 155 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 156 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 157 | -------------------------------------------------------------------------------- /analysis/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opentensor/validators/9e2172f3889faa9ae5bc7dbae08ba4291610a022/analysis/__init__.py -------------------------------------------------------------------------------- /analysis/demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import os\n", 10 | "import pandas as pd\n", 11 | "# set pandas column width\n", 12 | "pd.set_option('display.max_colwidth', 200)\n", 13 | "\n", 14 | "import openvalidators\n", 15 | "from utils import download_data, get_runs\n", 16 | "import plotting \n" 17 | ] 18 | }, 19 | { 20 | "attachments": {}, 21 | "cell_type": "markdown", 22 | "metadata": {}, 23 | "source": [ 24 | "# Download the wandb Data" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": null, 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "DEFAULT_FILTERS = {\"tags\": {\"$in\": [openvalidators.__version__]}}\n", 34 | "\n", 35 | "# download most recent runs from wandb\n", 36 | "runs = get_runs(filters=DEFAULT_FILTERS, return_paths=True)\n", 37 | "runs = runs[:30]\n", 38 | "print('\\nDownloading data for the following runs:')\n", 39 | "for i, run in enumerate(runs, start=1):\n", 40 | " print(f'{i:>3} - {run}')\n", 41 | " \n", 42 | "df = download_data(runs) \n", 43 | "\n", 44 | "# Print some info about the dataframe\n", 45 | "df.info()\n", 46 | "print(f'History for {len(runs)} runs contains {len(df)} records. Dataframe shape: {df.shape}')" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": null, 52 | "metadata": {}, 53 | "outputs": [], 54 | "source": [ 55 | "# Contents of first log entry\n", 56 | "df.iloc[0]" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": null, 62 | "metadata": {}, 63 | "outputs": [], 64 | "source": [ 65 | "# Get length of downloaded runs\n", 66 | "df.groupby('run_id').size()" 67 | ] 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "metadata": {}, 72 | "source": [ 73 | "# Network Health Overview\n", 74 | "The plot below shows the diversity of completions produced by each UID, which is defined as the number of unique completions divided by the total number of completions. A low diversity score indicates that the miner is producing the same completions over and over again, while a high diversity score indicates that the miner is producing a wide variety of completions. Good UIDs should be in the top right corner. Color indicates average reward for that UID" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": null, 80 | "metadata": {}, 81 | "outputs": [], 82 | "source": [ 83 | "\n", 84 | "plotting.plot_uid_diversty(df, remove_unsuccessful=True)" 85 | ] 86 | }, 87 | { 88 | "attachments": {}, 89 | "cell_type": "markdown", 90 | "metadata": {}, 91 | "source": [ 92 | "# Logging throughput\n", 93 | "The plot below shows the event logging rate" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": null, 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [ 102 | "plotting.plot_throughput(df,n_minutes=10)" 103 | ] 104 | }, 105 | { 106 | "attachments": {}, 107 | "cell_type": "markdown", 108 | "metadata": {}, 109 | "source": [ 110 | "# Dendrite Response Success Rates\n", 111 | "The plots below show the success rates of the dendrite responses. The success rate is defined as the number of completions that recieve nonzero reward." 112 | ] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "execution_count": null, 117 | "metadata": {}, 118 | "outputs": [], 119 | "source": [ 120 | "# plots dendrite completion success rates (for answers by default), default is 20 most queried uids\n", 121 | "plotting.plot_dendrite_rates(df)\n" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": null, 127 | "metadata": {}, 128 | "outputs": [], 129 | "source": [ 130 | "# plot dendrite completion rates for specific UID followups \n", 131 | "plotting.plot_dendrite_rates(df, uid_col='followup_uids', reward_col='followup_rewards', uids=[11,4,72,105,21,444,51,34,100,200,299])\n" 132 | ] 133 | }, 134 | { 135 | "attachments": {}, 136 | "cell_type": "markdown", 137 | "metadata": {}, 138 | "source": [ 139 | "# Analyze Completion Rates\n", 140 | "The plots below show the rate of popular completions produced by the network. These can help to detect spam/attacks.\n", 141 | "\n", 142 | "**Note**: Empty string responses are a result of unsuccessful completions." 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "execution_count": null, 148 | "metadata": {}, 149 | "outputs": [], 150 | "source": [ 151 | "# Show rate of 10 most popular followup completions \n", 152 | "plotting.plot_completion_rates(df, msg_col='followup_completions', time_interval='H', ntop=10).update_layout(yaxis_type='log')" 153 | ] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "execution_count": null, 158 | "metadata": {}, 159 | "outputs": [], 160 | "source": [ 161 | "# same, but for answers\n", 162 | "plotting.plot_completion_rates(df, msg_col='answer_completions', time_interval='H', ntop=10).update_layout(yaxis_type='log')" 163 | ] 164 | }, 165 | { 166 | "cell_type": "code", 167 | "execution_count": null, 168 | "metadata": {}, 169 | "outputs": [], 170 | "source": [ 171 | "# regex-based completion search\n", 172 | "plotting.plot_completion_rates(df, msg_col='answer_completions', time_interval='H', ntop=5, completion_regex='Beyoncé')" 173 | ] 174 | }, 175 | { 176 | "attachments": {}, 177 | "cell_type": "markdown", 178 | "metadata": {}, 179 | "source": [ 180 | "## Rewards for a Set of Completions\n", 181 | "The plots below show the rewards for a set of completions. This is a complement to the completion rates, as they show whether the completions are actually rewarded highly or not." 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "metadata": {}, 188 | "outputs": [], 189 | "source": [ 190 | "# plot reward rates for 5 most popular followup completions\n", 191 | "plotting.plot_completion_rewards(df, ntop=5, msg_col='followup_completions', reward_col='followup_rewards', uid_col='followup_uids')\n" 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": null, 197 | "metadata": {}, 198 | "outputs": [], 199 | "source": [ 200 | "# same, but for answers\n", 201 | "plotting.plot_completion_rewards(df, ntop=5, msg_col='answer_completions', reward_col='answer_rewards', uid_col='answer_uids')\n" 202 | ] 203 | }, 204 | { 205 | "attachments": {}, 206 | "cell_type": "markdown", 207 | "metadata": {}, 208 | "source": [ 209 | "# Leaderboards\n", 210 | "The plots below show the leaderboard of UIDs and completions, based on various factors." 211 | ] 212 | }, 213 | { 214 | "cell_type": "code", 215 | "execution_count": null, 216 | "metadata": {}, 217 | "outputs": [], 218 | "source": [ 219 | "# Get UIDs with highest average answer rewards \n", 220 | "plotting.plot_leaderboard(df, ntop=30)" 221 | ] 222 | }, 223 | { 224 | "cell_type": "code", 225 | "execution_count": null, 226 | "metadata": {}, 227 | "outputs": [], 228 | "source": [ 229 | "# Get answer completions with highest average rewards \n", 230 | "plotting.plot_leaderboard(df, ntop=10, group_on='answer_completions', agg_col='answer_rewards', alias=True)" 231 | ] 232 | }, 233 | { 234 | "cell_type": "code", 235 | "execution_count": null, 236 | "metadata": {}, 237 | "outputs": [], 238 | "source": [ 239 | "# Get followup completions with highest max reward \n", 240 | "plotting.plot_leaderboard(df, ntop=10, group_on='followup_completions', agg_col='followup_rewards', agg='max', alias=True)" 241 | ] 242 | }, 243 | { 244 | "cell_type": "code", 245 | "execution_count": null, 246 | "metadata": {}, 247 | "outputs": [], 248 | "source": [ 249 | "# Get answer completions with highest frequency\n", 250 | "plotting.plot_leaderboard(df, ntop=5, group_on='answer_completions', agg_col='answer_rewards', agg='size', alias=True)" 251 | ] 252 | } 253 | ], 254 | "metadata": { 255 | "kernelspec": { 256 | "display_name": "Python 3", 257 | "language": "python", 258 | "name": "python3" 259 | }, 260 | "language_info": { 261 | "codemirror_mode": { 262 | "name": "ipython", 263 | "version": 3 264 | }, 265 | "file_extension": ".py", 266 | "mimetype": "text/x-python", 267 | "name": "python", 268 | "nbconvert_exporter": "python", 269 | "pygments_lexer": "ipython3", 270 | "version": "3.9.5" 271 | } 272 | }, 273 | "nbformat": 4, 274 | "nbformat_minor": 2 275 | } 276 | -------------------------------------------------------------------------------- /analysis/utils.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | 18 | import os 19 | import tqdm 20 | import wandb 21 | import pandas as pd 22 | from pandas.api.types import is_list_like 23 | 24 | from typing import List, Dict, Any, Union 25 | 26 | 27 | def get_runs(project: str = "opentensor-dev/openvalidators", filters: Dict[str, Any] = None, return_paths: bool = False) -> List: 28 | """Download runs from wandb. 29 | 30 | Args: 31 | project (str): Name of the project. Defaults to 'openvalidators' (community project) 32 | filters (Dict[str, Any], optional): Optional run filters for wandb api. Defaults to None. 33 | return_paths (bool, optional): Return only run paths. Defaults to False. 34 | 35 | Returns: 36 | List[wandb.apis.public.Run]: List of runs or run paths (List[str]). 37 | """ 38 | api = wandb.Api() 39 | wandb.login(anonymous="allow") 40 | 41 | runs = api.runs(project, filters=filters) 42 | if return_paths: 43 | return [os.path.join(run.entity, run.project, run.id) for run in runs] 44 | else: 45 | return runs 46 | 47 | 48 | def download_data(run_path: Union[str, List] = None, timeout: float = 600) -> pd.DataFrame: 49 | """Download data from wandb. 50 | 51 | Args: 52 | run_path (Union[str, List], optional): Path to run or list of paths. Defaults to None. 53 | timeout (float, optional): Timeout for wandb api. Defaults to 600. 54 | 55 | Returns: 56 | pd.DataFrame: Dataframe of event log. 57 | """ 58 | api = wandb.Api(timeout=timeout) 59 | wandb.login(anonymous="allow") 60 | 61 | if isinstance(run_path, str): 62 | run_path = [run_path] 63 | 64 | frames = [] 65 | total_events = 0 66 | pbar = tqdm.tqdm(sorted(run_path), desc="Loading history from wandb", total=len(run_path), unit="run") 67 | for path in pbar: 68 | run = api.run(path) 69 | 70 | frame = pd.DataFrame(list(run.scan_history())) 71 | frames.append(frame.assign(run_id=path.split("/")[-1])) 72 | total_events += len(frame) 73 | 74 | pbar.set_postfix({"total_events": total_events}) 75 | 76 | df = pd.concat(frames) 77 | # Convert timestamp to datetime. 78 | df._timestamp = pd.to_datetime(df._timestamp, unit="s") 79 | df.sort_values("_timestamp", inplace=True) 80 | 81 | return df 82 | 83 | 84 | def load_data(path: str, nrows: int = None): 85 | """Load data from csv.""" 86 | df = pd.read_csv(path, nrows=nrows) 87 | 88 | # detect list columns which as stored as strings 89 | list_cols = [c for c in df.columns if df[c].dtype == "object" and df[c].str.startswith("[").all()] 90 | 91 | # convert string representation of list to list 92 | df[list_cols] = df[list_cols].applymap(eval) 93 | 94 | return df 95 | 96 | 97 | def explode_data(df: pd.DataFrame, list_cols: List[str] = None, list_len: int = None) -> pd.DataFrame: 98 | """Explode list columns in dataframe so that each element in the list is a separate row. 99 | 100 | Args: 101 | df (pd.DataFrame): Dataframe of event log. 102 | list_cols (List[str], optional): List of columns to explode. Defaults to None. 103 | list_len (int, optional): Length of list. Defaults to None. 104 | 105 | Returns: 106 | pd.DataFrame: Dataframe with exploded list columns. 107 | """ 108 | if list_cols is None: 109 | list_cols = [c for c in df.columns if df[c].apply(is_list_like).all()] 110 | print(f"Exploding {len(list_cols)}) list columns with {list_len} elements: {list_cols}") 111 | if list_len: 112 | list_cols = [c for c in list_cols if df[c].apply(len).unique()[0] == list_len] 113 | print(f"Exploding {len(list_cols)}) list columns with {list_len} elements: {list_cols}") 114 | 115 | return df.explode(column=list_cols) 116 | 117 | 118 | def get_list_col_lengths(df: pd.DataFrame) -> Dict[str, int]: 119 | """Helper function to get the length of list columns.""" 120 | list_col_lengths = {c: sorted(df[c].apply(len).unique()) for c in df.columns if df[c].apply(is_list_like).all()} 121 | varying_lengths = {c: v for c, v in list_col_lengths.items() if len(v) > 1} 122 | 123 | if len(varying_lengths) > 0: 124 | print(f"The following columns have varying lengths: {varying_lengths}") 125 | 126 | return {c: v[0] for c, v in list_col_lengths.items()} 127 | -------------------------------------------------------------------------------- /openvalidators/__init__.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | from . import config 18 | from . import dendrite 19 | from . import forward 20 | from . import gating 21 | from . import misc 22 | from . import mock 23 | from . import neuron 24 | from . import prompts 25 | from . import reward 26 | from . import run 27 | from . import utils 28 | from . import weights 29 | from . import event 30 | 31 | __version__ = "1.2.0" 32 | version_split = __version__.split(".") 33 | __spec_version__ = (1000 * int(version_split[0])) + (10 * int(version_split[1])) + (1 * int(version_split[2])) 34 | -------------------------------------------------------------------------------- /openvalidators/dataset.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | 18 | import random 19 | import bittensor as bt 20 | from datasets import load_dataset 21 | from collections.abc import Iterator 22 | 23 | class Dataset(Iterator): 24 | def __init__(self): 25 | super().__init__() 26 | seed = random.randint(0,1000) 27 | self.openwebtext = iter( load_dataset("openwebtext", split="train", streaming=True).shuffle(seed=seed, buffer_size=10000) ) 28 | self.red_pajama = iter( load_dataset("togethercomputer/RedPajama-Data-1T", 'default', split='train', streaming=True).shuffle(seed=seed, buffer_size=10000) ) 29 | 30 | def __next__(self): 31 | while True: 32 | bt.logging.debug('Retrieving data from dataset...') 33 | if random.random() < 0.5: 34 | text = next(self.openwebtext)["text"] 35 | else: 36 | text = next(self.red_pajama)["text"] 37 | 38 | # Check if the text is not empty or does not consist only of newline characters 39 | if text.strip(): 40 | return {"text": text} 41 | 42 | 43 | class MockDataset(Iterator): 44 | def __next__(self): 45 | return {"text": "What is the capital of Texas?"} 46 | 47 | 48 | -------------------------------------------------------------------------------- /openvalidators/dendrite.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | 18 | import torch 19 | import asyncio 20 | import bittensor as bt 21 | from typing import List 22 | 23 | 24 | class AsyncDendritePool: 25 | def __init__(self, keypair, metagraph): 26 | self.dendrites = [bt.text_prompting(axon=axon, keypair=keypair, uid=uid) for uid, axon in enumerate(metagraph.axons)] 27 | 28 | async def async_forward( 29 | self, 30 | uids: List[int], 31 | roles: List[str], 32 | messages: List[str], 33 | timeout: float = 12, 34 | return_call=True, 35 | ): 36 | # The following asyncio definition queries a single endpoint with the message 37 | # prompt and returns the response. 38 | def call_single_uid(uid: int): 39 | return self.dendrites[uid].async_forward(roles=roles, messages=messages, return_call=return_call, timeout=timeout) 40 | 41 | # The following asyncio definition gathers the responses 42 | # from multiple coroutines for each uid. 43 | async def query(): 44 | coroutines = [call_single_uid(uid) for uid in uids] 45 | all_responses = await asyncio.gather(*coroutines) 46 | return all_responses 47 | 48 | return await query() 49 | 50 | async def async_backward( 51 | self, 52 | uids: List[int], 53 | roles: List[str], 54 | messages: List[str], 55 | completions: List[str], 56 | rewards: torch.FloatTensor, 57 | timeout: float = 1, # response not required 58 | ): 59 | # Async call single endpoint with reward for its completion. 60 | def call_single_uid(uid: int, completion: str, reward: torch.Tensor): 61 | return self.dendrites[uid].async_backward( 62 | roles=roles, messages=messages, completion=completion, rewards=[reward.item()], timeout=timeout 63 | ) 64 | 65 | # The following asyncio definition gathers the responses 66 | # from multiple coroutines for each uid. 67 | async def query(): 68 | coroutines = [ 69 | call_single_uid(uid, completion, reward) for uid, completion, reward in zip(uids, completions, rewards) 70 | ] 71 | all_responses = await asyncio.gather(*coroutines) 72 | return all_responses 73 | 74 | return await query() 75 | 76 | def resync(self, metagraph: "bt.metagraph.Metagraph"): 77 | """ 78 | Resyncs the dendrites with the latest updated version of the metagraph. 79 | 80 | Parameters: 81 | metagraph (:`bittensor.metagraph.Metagraph`): latest updated version of the metagraph. 82 | """ 83 | metagraph_uids_axons_state = dict(zip(metagraph.uids.tolist(), metagraph.axons)) 84 | 85 | # Iterate over all dendrites and check if the metagraph has updated. 86 | for index, dendrite in enumerate(self.dendrites): 87 | current_uid_axon_info = metagraph_uids_axons_state.pop(dendrite.uid) 88 | 89 | if dendrite.axon_info != current_uid_axon_info: 90 | self.dendrites[index] = bt.text_prompting( 91 | axon=current_uid_axon_info, 92 | keypair=dendrite.keypair, 93 | uid=dendrite.uid, 94 | ) 95 | 96 | # If there are any new axons, add them to the dendrites. 97 | if len(metagraph_uids_axons_state) > 0: 98 | for uid, axon in metagraph_uids_axons_state.items(): 99 | self.dendrites.append(bt.text_prompting(axon=axon, keypair=self.dendrites[0].keypair, uid=uid)) 100 | -------------------------------------------------------------------------------- /openvalidators/event.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | 18 | import bittensor as bt 19 | from dataclasses import dataclass 20 | from typing import List, Optional 21 | 22 | from openvalidators.reward import RewardModelType 23 | 24 | 25 | @dataclass 26 | class EventSchema: 27 | completions: List[str] # List of completions received for a given prompt 28 | completion_times: List[float] # List of completion times for a given prompt 29 | name: str # Prompt type, e.g. 'followup', 'answer' 30 | block: float # Current block at given step 31 | gating_loss: float # Gating model loss for given step 32 | uids: List[int] # Queried uids 33 | prompt: str # Prompt text string 34 | step_length: float # Elapsed time between the beginning of a run step to the end of a run step 35 | best: str # Best completion for given prompt 36 | 37 | # Reward data 38 | rewards: List[float] # Reward vector for given step 39 | dahoas_reward_model: Optional[List[float]] # Output vector of the dahoas reward model 40 | blacklist_filter: Optional[List[float]] # Output vector of the blacklist filter 41 | nsfw_filter: Optional[List[float]] # Output vector of the nsfw filter 42 | reciprocate_reward_model: Optional[List[float]] # Output vector of the reciprocate reward model 43 | diversity_reward_model: Optional[List[float]] # Output vector of the diversity reward model 44 | dpo_reward_model: Optional[List[float]] # Output vector of the dpo reward model 45 | rlhf_reward_model: Optional[List[float]] # Output vector of the rlhf reward model 46 | prompt_reward_model: Optional[List[float]] # Output vector of the prompt reward model 47 | relevance_filter: Optional[List[float]] # Output vector of the relevance scoring reward model 48 | task_validator_filter: Optional[List[float]] 49 | 50 | dahoas_reward_model_normalized: Optional[List[float]] # Output vector of the dahoas reward model 51 | nsfw_filter_normalized: Optional[List[float]] # Output vector of the nsfw filter 52 | reciprocate_reward_model_normalized: Optional[List[float]] # Output vector of the reciprocate reward model 53 | diversity_reward_model_normalized: Optional[List[float]] # Output vector of the diversity reward model 54 | dpo_reward_model_normalized: Optional[List[float]] # Output vector of the dpo reward model 55 | rlhf_reward_model_normalized: Optional[List[float]] # Output vector of the rlhf reward model 56 | prompt_reward_model_normalized: Optional[List[float]] # Output vector of the prompt reward model 57 | relevance_filter_normalized: Optional[List[float]] # Output vector of the relevance scoring reward model 58 | task_validator_filter_normalized: Optional[List[float]] 59 | 60 | # Weights data 61 | set_weights: Optional[List[List[float]]] 62 | 63 | @staticmethod 64 | def from_dict(event_dict: dict, disable_log_rewards: bool) -> 'EventSchema': 65 | """Converts a dictionary to an EventSchema object.""" 66 | rewards = { 67 | 'blacklist_filter': event_dict.get(RewardModelType.blacklist.value), 68 | 'dahoas_reward_model': event_dict.get(RewardModelType.dahoas.value), 69 | 'task_validator_filter': event_dict.get(RewardModelType.task_validator.value), 70 | 'nsfw_filter': event_dict.get(RewardModelType.nsfw.value), 71 | 'relevance_filter': event_dict.get(RewardModelType.relevance.value), 72 | 'reciprocate_reward_model': event_dict.get(RewardModelType.reciprocate.value), 73 | 'diversity_reward_model': event_dict.get(RewardModelType.diversity.value), 74 | 'dpo_reward_model': event_dict.get(RewardModelType.dpo.value), 75 | 'rlhf_reward_model': event_dict.get(RewardModelType.rlhf.value), 76 | 'prompt_reward_model': event_dict.get(RewardModelType.prompt.value), 77 | 78 | 'dahoas_reward_model_normalized': event_dict.get(RewardModelType.dahoas.value + '_normalized'), 79 | 'task_validator_filter_normalized': event_dict.get(RewardModelType.task_validator.value + '_normalized'), 80 | 'nsfw_filter_normalized': event_dict.get(RewardModelType.nsfw.value + '_normalized'), 81 | 'relevance_filter_normalized': event_dict.get(RewardModelType.relevance.value + '_normalized'), 82 | 'reciprocate_reward_model_normalized': event_dict.get(RewardModelType.reciprocate.value + '_normalized'), 83 | 'diversity_reward_model_normalized': event_dict.get(RewardModelType.diversity.value + '_normalized'), 84 | 'dpo_reward_model_normalized': event_dict.get(RewardModelType.dpo.value + '_normalized'), 85 | 'rlhf_reward_model_normalized': event_dict.get(RewardModelType.rlhf.value + '_normalized'), 86 | 'prompt_reward_model_normalized': event_dict.get(RewardModelType.prompt.value + '_normalized'), 87 | } 88 | 89 | # Logs warning that expected data was not set properly 90 | if not disable_log_rewards and any(value is None for value in rewards.values()): 91 | for key, value in rewards.items(): 92 | if value is None: 93 | bt.logging.warning(f'EventSchema.from_dict: {key} is None, data will not be logged') 94 | 95 | return EventSchema( 96 | completions=event_dict['completions'], 97 | completion_times=event_dict['completion_times'], 98 | name=event_dict['name'], 99 | block=event_dict['block'], 100 | gating_loss=event_dict['gating_loss'], 101 | uids=event_dict['uids'], 102 | prompt=event_dict['prompt'], 103 | step_length=event_dict['step_length'], 104 | best=event_dict['best'], 105 | rewards=event_dict['rewards'], 106 | **rewards, 107 | set_weights=None, 108 | ) 109 | -------------------------------------------------------------------------------- /openvalidators/forward.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN 17 | # THE SOFTWARE. 18 | 19 | import time 20 | import torch 21 | import random 22 | import bittensor as bt 23 | import random 24 | 25 | from loguru import logger 26 | from typing import List 27 | from dataclasses import asdict 28 | from openvalidators.event import EventSchema 29 | from openvalidators.misc import ttl_get_block 30 | from openvalidators.prompts import followup_prompt, answer_prompt, augment_prompt 31 | from openvalidators.utils import check_uid_availability 32 | 33 | 34 | def get_random_uids(self, k: int, exclude: List[int] = None) -> torch.LongTensor: 35 | """Returns k available random uids from the metagraph. 36 | Args: 37 | k (int): Number of uids to return. 38 | exclude (List[int]): List of uids to exclude from the random sampling. 39 | Returns: 40 | uids (torch.LongTensor): Randomly sampled available uids. 41 | Notes: 42 | If `k` is larger than the number of available `uids`, set `k` to the number of available `uids`. 43 | """ 44 | candidate_uids = [] 45 | avail_uids = [] 46 | 47 | for uid in range(self.metagraph.n.item()): 48 | uid_is_available = check_uid_availability(self.metagraph, uid, self.config.neuron.vpermit_tao_limit) 49 | uid_is_not_excluded = exclude is None or uid not in exclude 50 | 51 | if uid_is_available: 52 | avail_uids.append(uid) 53 | if uid_is_not_excluded: 54 | candidate_uids.append(uid) 55 | 56 | # Check if candidate_uids contain enough for querying, if not grab all avaliable uids 57 | available_uids = candidate_uids 58 | if len(candidate_uids) < k: 59 | available_uids += random.sample([uid for uid in avail_uids if uid not in candidate_uids], k-len(candidate_uids)) 60 | 61 | uids = torch.tensor(random.sample(available_uids, k), dtype=torch.int64) 62 | return uids 63 | 64 | 65 | async def run_step(self, prompt: str, k: int, timeout: float, name: str, exclude: list = [], base_prompt = None): 66 | 67 | if base_prompt == None: 68 | base_prompt = prompt 69 | 70 | bt.logging.debug("run_step", name) 71 | 72 | # Record event start time. 73 | event = {"name": name} 74 | start_time = time.time() 75 | 76 | # Get the list of uids to query for this step. 77 | uids = get_random_uids(self, k=k, exclude=exclude).to(self.device) 78 | 79 | # Make calls to the network with the prompt. 80 | responses: List[bt.DendriteCall] = await self.dendrite_pool.async_forward( 81 | uids=uids, 82 | roles=["user"], 83 | messages=[prompt], 84 | timeout=timeout, 85 | ) 86 | 87 | # Compute the rewards for the responses given the prompt. 88 | rewards: torch.FloatTensor = torch.zeros(len(responses), dtype=torch.float32).to(self.device) 89 | for weight_i, reward_fn_i in zip(self.reward_weights, self.reward_functions): 90 | reward_i, reward_i_normalized = reward_fn_i.apply(prompt, responses, name) 91 | rewards += weight_i * reward_i_normalized.to(self.device) 92 | if not self.config.neuron.disable_log_rewards: 93 | event[reward_fn_i.name] = reward_i.tolist() 94 | event[reward_fn_i.name + '_normalized'] = reward_i_normalized.tolist() 95 | bt.logging.trace(str(reward_fn_i.name), reward_i_normalized.tolist()) 96 | 97 | for masking_fn_i in self.masking_functions: 98 | mask_i, mask_i_normalized = masking_fn_i.apply(base_prompt, responses, name) 99 | rewards *= mask_i_normalized.to(self.device) # includes diversity 100 | if not self.config.neuron.disable_log_rewards: 101 | event[masking_fn_i.name] = mask_i.tolist() 102 | event[masking_fn_i.name + '_normalized'] = mask_i_normalized.tolist() 103 | bt.logging.trace(str(masking_fn_i.name), mask_i_normalized.tolist()) 104 | 105 | # Train the gating model based on the predicted scores and the actual rewards. 106 | gating_scores: torch.FloatTensor = self.gating_model(prompt).to(self.device) 107 | gating_loss: torch.FloatTensor = self.gating_model.backward(scores=gating_scores[uids], rewards=rewards) 108 | 109 | # Find the best completion given the rewards vector. 110 | completions: List[str] = [comp.completion for comp in responses] 111 | best: str = completions[rewards.argmax(dim=0)].strip() 112 | 113 | # Get completion times 114 | completion_times: List[float] = [comp.elapsed_time for comp in responses] 115 | 116 | # Compute forward pass rewards, assumes followup_uids and answer_uids are mutually exclusive. 117 | # shape: [ metagraph.n ] 118 | scattered_rewards: torch.FloatTensor = self.moving_averaged_scores.scatter(0, uids, rewards).to(self.device) 119 | 120 | # Update moving_averaged_scores with rewards produced by this step. 121 | # shape: [ metagraph.n ] 122 | alpha: float = self.config.neuron.moving_average_alpha 123 | self.moving_averaged_scores: torch.FloatTensor = alpha * scattered_rewards + (1 - alpha) * self.moving_averaged_scores.to( 124 | self.device 125 | ) 126 | 127 | # Log the step event. 128 | event.update( 129 | { 130 | "block": ttl_get_block(self), 131 | "step_length": time.time() - start_time, 132 | "prompt": prompt, 133 | "uids": uids.tolist(), 134 | "completions": completions, 135 | "completion_times": completion_times, 136 | "rewards": rewards.tolist(), 137 | "gating_loss": gating_loss.item(), 138 | "best": best, 139 | } 140 | ) 141 | bt.logging.debug("event:", str(event)) 142 | if not self.config.neuron.dont_save_events: 143 | logger.log("EVENTS", "events", **event) 144 | 145 | # Log the event to wandb. 146 | wandb_event = EventSchema.from_dict(event, self.config.neuron.disable_log_rewards) 147 | if not self.config.wandb.off: 148 | self.wandb.log(asdict(wandb_event)) 149 | 150 | # Return the event. 151 | return event 152 | 153 | 154 | async def forward(self): 155 | 156 | # Obtain a unique context from the dataset. 157 | data = next(self.dataset)["text"] 158 | 159 | random_cutoff = random.randint(15, 30) 160 | # Truncate context to a limited set of sentences. 161 | base_text = ".".join(data.split(".", maxsplit=random_cutoff)[:-1]) 162 | aug_prompt = augment_prompt(base_text) 163 | 164 | # Reset Blacklist reward model 165 | self.blacklist.reset() 166 | 167 | # Request a summary, given the original context. 168 | augment_event = await run_step( 169 | self, 170 | prompt=aug_prompt, 171 | name="augment", 172 | k=self.config.neuron.followup_sample_size, 173 | timeout=self.config.neuron.followup_timeout, 174 | ) 175 | 176 | base_text = augment_event["best"] 177 | base_prompt = augment_event["best"] 178 | exclude = augment_event["uids"] 179 | for k in range(self.config.neuron.num_followup_steps): 180 | 181 | # Get a followup question, given the summarized context. 182 | prompt = followup_prompt(base_text, i=k) 183 | followup_event = await run_step( 184 | self, 185 | prompt=prompt, 186 | name="followup" + str(k), 187 | k=self.config.neuron.followup_sample_size, 188 | timeout=self.config.neuron.followup_timeout, 189 | exclude=exclude, 190 | base_prompt=base_prompt 191 | ) 192 | exclude += followup_event["uids"] 193 | 194 | # Ask the followup question, given the original context. 195 | prompt = answer_prompt(base_text, followup_event["best"]) 196 | answer_event = await run_step( 197 | self, 198 | prompt=prompt, 199 | name="answer" + str(k), 200 | k=self.config.neuron.answer_sample_size, 201 | timeout=self.config.neuron.answer_timeout, 202 | exclude=exclude, 203 | base_prompt=followup_event["best"] 204 | ) 205 | exclude += answer_event["uids"] 206 | 207 | self.blacklist.question_blacklist.append(followup_event["best"]) 208 | self.blacklist.answer_blacklist.append(answer_event["best"]) 209 | 210 | if k == 0: 211 | # Extend the base text with the best answer. 212 | base_text = ( 213 | base_text + "\nPrevious Question \nQuestion:" + followup_event["best"] + "\nAnswer:" + answer_event["best"] 214 | ) 215 | else: 216 | base_text = base_text + "\nQuestion:" + followup_event["best"] + "\nAnswer:" + answer_event["best"] 217 | -------------------------------------------------------------------------------- /openvalidators/misc.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | 18 | import time 19 | import math 20 | import hashlib as rpccheckhealth 21 | from math import floor 22 | from typing import Callable, Any 23 | from functools import lru_cache, update_wrapper 24 | # LRU Cache with TTL 25 | def ttl_cache(maxsize: int = 128, typed: bool = False, ttl: int = -1): 26 | if ttl <= 0: 27 | ttl = 65536 28 | hash_gen = _ttl_hash_gen(ttl) 29 | 30 | def wrapper(func: Callable) -> Callable: 31 | @lru_cache(maxsize, typed) 32 | def ttl_func(ttl_hash, *args, **kwargs): 33 | return func(*args, **kwargs) 34 | 35 | def wrapped(*args, **kwargs) -> Any: 36 | th = next(hash_gen) 37 | return ttl_func(th, *args, **kwargs) 38 | 39 | return update_wrapper(wrapped, func) 40 | 41 | return wrapper 42 | 43 | 44 | def _ttl_hash_gen(seconds: int): 45 | start_time = time.time() 46 | while True: 47 | yield floor((time.time() - start_time) / seconds) 48 | 49 | 50 | # 12 seconds updating block. 51 | @ttl_cache(maxsize=1, ttl=12) 52 | def ttl_get_block(self) -> int: 53 | return self.subtensor.get_current_block() 54 | -------------------------------------------------------------------------------- /openvalidators/mock.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | 18 | import torch 19 | import asyncio 20 | import bittensor as bt 21 | from openvalidators.prompts import FirewallPrompt, FollowupPrompt, AnswerPrompt 22 | from openvalidators.gating import BaseGatingModel 23 | from typing import List 24 | 25 | 26 | class MockGatingModel(BaseGatingModel): 27 | def __init__(self, num_uids: int): 28 | super(MockGatingModel, self).__init__() 29 | # super(MockGatingModel, self).__init__() 30 | self.num_uids = num_uids 31 | self.linear = torch.nn.Linear(256, 10) 32 | 33 | def forward(self, message: str) -> "torch.FloatTensor": 34 | return torch.randn(self.num_uids) 35 | 36 | def backward(self, scores: torch.FloatTensor, rewards: torch.FloatTensor): 37 | return torch.tensor(0.0) 38 | 39 | def resync( 40 | self, 41 | previous_metagraph: "bt.metagraph.Metagraph", 42 | metagraph: "bt.metagraph.Metagraph", 43 | ): 44 | pass 45 | 46 | 47 | class MockRewardModel(torch.nn.Module): 48 | def reward( 49 | self, 50 | completions_with_prompt: List[str], 51 | completions_without_prompt: List[str], 52 | difference=False, 53 | shift=3, 54 | ) -> torch.FloatTensor: 55 | return torch.zeros(len(completions_with_prompt)) 56 | 57 | 58 | class MockDendriteResponse: 59 | completion = "" 60 | elapsed_time = 0 61 | is_success = True 62 | firewall_prompt = FirewallPrompt() 63 | followup_prompt = FollowupPrompt() 64 | answer_prompt = AnswerPrompt() 65 | 66 | def __init__(self, message: str): 67 | if self.firewall_prompt.matches_template(message): 68 | self.completion = self.firewall_prompt.mock_response() 69 | elif self.followup_prompt.matches_template(message): 70 | self.completion = self.followup_prompt.mock_response() 71 | elif self.answer_prompt.matches_template(message): 72 | self.completion = self.answer_prompt.mock_response() 73 | else: 74 | self.completion = "The capital of Texas is Austin." 75 | 76 | def __str__(self): 77 | return f"MockDendriteResponse({self.completion})" 78 | 79 | def __repr__(self): 80 | return f"MockDendriteResponse({self.completion})" 81 | 82 | 83 | class MockDendritePool(torch.nn.Module): 84 | def forward(self, roles: List[str], messages: List[str], uids: List[int], timeout: float): 85 | return [MockDendriteResponse(messages[0]) for _ in uids] 86 | 87 | async def async_forward( 88 | self, 89 | roles: List[str], 90 | messages: List[str], 91 | uids: List[int], 92 | timeout: float = 12, 93 | return_call=True, 94 | ): 95 | async def query(): 96 | await asyncio.sleep(0.01) 97 | return [MockDendriteResponse(messages[0]) for _ in uids] 98 | 99 | return await query() 100 | 101 | def resync(self, metagraph): 102 | pass 103 | 104 | async def async_backward( 105 | self, uids: List[int], roles: List[str], messages: List[str], completions: List[str], rewards: List[float] 106 | ): 107 | async def query(): 108 | await asyncio.sleep(0.01) 109 | return [MockDendriteResponse(messages[0]) for _ in uids] 110 | 111 | return await query() 112 | -------------------------------------------------------------------------------- /openvalidators/reward/__init__.py: -------------------------------------------------------------------------------- 1 | from .blacklist import Blacklist 2 | from .task_validator import TaskValidator 3 | from .nsfw import NSFWRewardModel 4 | from .dpo import DirectPreferenceRewardModel 5 | from .open_assistant import OpenAssistantRewardModel 6 | from .reciprocate import ReciprocateRewardModel 7 | from .relevance import RelevanceRewardModel 8 | from .reward import BaseRewardModel 9 | from .reward import MockRewardModel 10 | from .dahoas import DahoasRewardModel 11 | from .diversity import DiversityRewardModel 12 | from .prompt import PromptRewardModel 13 | from .config import RewardModelType, DefaultRewardFrameworkConfig -------------------------------------------------------------------------------- /openvalidators/reward/blacklist.py: -------------------------------------------------------------------------------- 1 | 2 | # The MIT License (MIT) 3 | # Copyright © 2021 Yuma Rao 4 | 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 6 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 7 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 8 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 9 | 10 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 11 | # the Software. 12 | 13 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 14 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 15 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 16 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 17 | # DEALINGS IN THE SOFTWARE. 18 | 19 | import torch 20 | from typing import List 21 | from .config import RewardModelType 22 | from .reward import BaseRewardModel 23 | 24 | blacklist = ["That is an excellent question."] 25 | 26 | class Blacklist( BaseRewardModel ): 27 | 28 | @property 29 | def name(self) -> str: return RewardModelType.blacklist.value 30 | 31 | def __init__(self): 32 | super().__init__() 33 | self.question_blacklist = [] 34 | self.answer_blacklist = [] 35 | 36 | def reward( self, prompt: str, completion: str, name: str ) -> float: 37 | if completion in blacklist: 38 | return 0.0 39 | 40 | if completion == prompt: 41 | return 0.0 42 | 43 | if completion in self.question_blacklist or completion in self.answer_blacklist: 44 | return 0.0 45 | 46 | return 1 47 | 48 | def get_rewards( self, prompt: str, completions: List[str], name: str ) -> torch.FloatTensor: 49 | return torch.tensor( [self.reward( prompt, completion, name ) for completion in completions], dtype=torch.float32) 50 | 51 | def normalize_rewards( self, rewards: torch.FloatTensor ) -> torch.FloatTensor: 52 | return rewards 53 | 54 | def reset(self): 55 | self.question_blacklist = [] 56 | self.answer_blacklist = [] -------------------------------------------------------------------------------- /openvalidators/reward/config.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 5 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 6 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 9 | # the Software. 10 | 11 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 12 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 13 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 15 | # DEALINGS IN THE SOFTWARE. 16 | from dataclasses import dataclass 17 | from enum import Enum 18 | 19 | 20 | class RewardModelType(Enum): 21 | dpo = 'dpo_reward_model' 22 | rlhf = 'rlhf_reward_model' 23 | reciprocate = 'reciprocate_reward_model' 24 | dahoas = 'dahoas_reward_model' 25 | diversity = 'diversity_reward_model' 26 | prompt = 'prompt_reward_model' 27 | blacklist = 'blacklist_filter' 28 | nsfw = 'nsfw_filter' 29 | relevance = 'relevance_filter' 30 | task_validator = 'task_validator_filter' 31 | 32 | 33 | @dataclass(frozen=True) 34 | class DefaultRewardFrameworkConfig: 35 | """Reward framework default configuration. 36 | Note: All the weights should add up to 1.0. 37 | """ 38 | dpo_model_weight: float = 0.3 39 | rlhf_model_weight: float = 0.4 40 | reciprocate_model_weight: float = 0.3 41 | dahoas_model_weight: float = 0 42 | prompt_model_weight: float = 0 43 | -------------------------------------------------------------------------------- /openvalidators/reward/dahoas.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | 18 | import os 19 | import torch 20 | from typing import List 21 | from .config import RewardModelType 22 | from .reward import BaseRewardModel 23 | from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig 24 | 25 | class DahoasRewardModel( BaseRewardModel ): 26 | 27 | model_name = "EleutherAI/gpt-j-6b" 28 | 29 | @property 30 | def name(self) -> str: return RewardModelType.dahoas.value 31 | 32 | @staticmethod 33 | def load_weights( path: str ): 34 | if not os.path.exists( path + "/hf_ckpt.pt"): 35 | os.makedirs( path, exist_ok=True) 36 | os.system( 37 | f"wget -O { path + '/hf_ckpt.pt'} \ 38 | https://huggingface.co/Dahoas/gptj-rm-static/resolve/main/hf_ckpt.pt" 39 | ) 40 | 41 | def __init__(self, path: str, device: str ): 42 | super().__init__() 43 | DahoasRewardModel.load_weights( path = path ) 44 | self.device = torch.device(device) 45 | config = AutoConfig.from_pretrained( DahoasRewardModel.model_name ) 46 | self.model = AutoModelForCausalLM.from_config( config ).to(self.device) 47 | self.config = self.model.config 48 | 49 | # `gpt-neo(x)` models use `hidden_size` attribute names instead of `n_embd`` 50 | if config is None: 51 | config = DahoasRewardModel.config() 52 | 53 | self.config.n_embd = self.config.hidden_size if hasattr(self.config, "hidden_size") else self.config.n_embd 54 | self.transformer = self.model.transformer 55 | self.v_head = torch.nn.Linear(self.config.n_embd, 1, bias=False).to(self.device) 56 | self.tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-j-6b") 57 | self.tokenizer.pad_token = self.tokenizer.eos_token 58 | self.PAD_ID = self.tokenizer(self.tokenizer.pad_token)["input_ids"][0] 59 | 60 | 61 | def reward( self, prompt: str, completion: str, name: str ) -> float: 62 | 63 | def reward_fn(samples): 64 | if samples is None: 65 | return 0 66 | scores_list = [] 67 | batch_size = 1 68 | for i in range(0, len(samples), batch_size): 69 | sub_samples = samples[i : i + batch_size] 70 | sub_samples = ["<|startoftext|>" + chosen + "<|endoftext|>" for chosen in sub_samples] 71 | encodings_dict = self.tokenizer( 72 | sub_samples, 73 | truncation=False, 74 | max_length=550, 75 | padding="max_length", 76 | return_tensors="pt", 77 | ) 78 | input_ids = encodings_dict["input_ids"].to(self.device) 79 | attn_masks = encodings_dict["attention_mask"].to(self.device) 80 | input_ids = input_ids.repeat(2, 1) 81 | attn_masks = attn_masks.repeat(2, 1) 82 | with torch.no_grad(): 83 | sub_scores = self.forward( 84 | input_ids = input_ids.to(self.device), 85 | attention_mask = attn_masks.to(self.device), 86 | ) 87 | scores_list.append(sub_scores["chosen_end_scores"]) 88 | scores = torch.cat(scores_list, dim=0).mean().item() 89 | return scores 90 | 91 | with torch.no_grad(): 92 | combined_reward = reward_fn( prompt + completion ) 93 | independent_reward = reward_fn( completion ) 94 | return float( (combined_reward - independent_reward).item() ) 95 | 96 | def get_rewards( self, prompt: str, completions: List[str], name: str ) -> torch.FloatTensor: 97 | return torch.tensor( [self.reward( prompt, completion, name ) for completion in completions], dtype=torch.float32).to(self.device) 98 | 99 | def forward( 100 | self, 101 | input_ids=None, 102 | past_key_values=None, 103 | attention_mask=None, 104 | token_type_ids=None, 105 | position_ids=None, 106 | head_mask=None, 107 | inputs_embeds=None, 108 | mc_token_ids=None, 109 | labels=None, 110 | return_dict=False, 111 | output_attentions=False, 112 | output_hidden_states=False, 113 | ): 114 | loss = None 115 | transformer_outputs = self.transformer( 116 | input_ids.to(self.device), 117 | attention_mask = attention_mask.to(self.device), 118 | ) 119 | 120 | hidden_states = transformer_outputs[0] 121 | 122 | rewards = self.v_head(hidden_states).squeeze(-1) 123 | chosen_end_scores = [] 124 | rejected_end_scores = [] 125 | 126 | # Split the inputs and rewards into two parts, chosen and rejected 127 | assert len(input_ids.shape) == 2 128 | bs = input_ids.shape[0] // 2 129 | chosen = input_ids[:bs] 130 | rejected = input_ids[bs:] 131 | chosen_rewards = rewards[:bs] 132 | rejected_rewards = rewards[bs:] 133 | 134 | loss = 0 135 | inference = False 136 | for i in range(bs): 137 | if torch.all(torch.eq(chosen[i], rejected[i])).item(): 138 | c_inds = (chosen[i] == self.PAD_ID).nonzero() 139 | c_ind = c_inds[0].item() if len(c_inds) > 0 else chosen.shape[1] 140 | chosen_end_scores.append(chosen_rewards[i, c_ind - 1]) 141 | inference = True 142 | continue 143 | 144 | # Check if there is any padding otherwise take length of sequence 145 | c_inds = (chosen[i] == self.PAD_ID).nonzero() 146 | c_ind = c_inds[0].item() if len(c_inds) > 0 else chosen.shape[1] 147 | r_inds = (rejected[i] == self.PAD_ID).nonzero() 148 | r_ind = r_inds[0].item() if len(r_inds) > 0 else rejected.shape[1] 149 | end_ind = max(c_ind, r_ind) 150 | 151 | # Retrieve first index where trajectories diverge 152 | divergence_ind = (chosen[i] != rejected[i]).nonzero()[0] 153 | assert divergence_ind > 0 154 | 155 | # Index into the correct rewards 156 | c_truncated_reward = chosen_rewards[i][divergence_ind:end_ind] 157 | r_truncated_reward = rejected_rewards[i][divergence_ind:end_ind] 158 | 159 | # Append the last rewards to the list of end scores 160 | chosen_end_scores.append(c_truncated_reward[-1]) 161 | rejected_end_scores.append(r_truncated_reward[-1]) 162 | 163 | # Compute loss based on truncated rewards (ignore padding) 164 | loss += -torch.log(torch.sigmoid(c_truncated_reward - r_truncated_reward)).mean() 165 | loss = loss / bs 166 | 167 | if not inference: 168 | chosen_end_scores = torch.stack(chosen_end_scores) 169 | rejected_end_scores = torch.stack(rejected_end_scores) 170 | 171 | if inference: 172 | chosen_end_scores = torch.stack(chosen_end_scores) 173 | return {"chosen_end_scores": chosen_end_scores} 174 | 175 | return { 176 | "loss": loss, 177 | "chosen_end_scores": chosen_end_scores, 178 | "rejected_end_scores": rejected_end_scores, 179 | } -------------------------------------------------------------------------------- /openvalidators/reward/diversity.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | 18 | import torch 19 | import torch.nn.functional as F 20 | from typing import List 21 | from .config import RewardModelType 22 | from .reward import BaseRewardModel 23 | from transformers import AutoTokenizer, AutoModel 24 | 25 | from torchmetrics.functional import pairwise_cosine_similarity 26 | 27 | def mean_pooling( model_output, attention_mask ): 28 | """Applies mean pooling to the token embeddings generated by the model. 29 | Args: 30 | model_output (torch.Tensor): Embedding model output, where the first element contains token embeddings. 31 | attention_mask (torch.Tensor): Attention mask to indicate valid tokens. 32 | Returns: 33 | torch.Tensor: Mean-pooled representation of the token embeddings. 34 | Notes: 35 | - The function calculates the mean-pooled representation using the attention mask for valid tokens. 36 | - Input_mask_expanded is created by expanding the attention mask to match the size of token embeddings. 37 | - The result is obtained by summing the element-wise multiplication of embeddings and input_mask_expanded, 38 | and dividing it by the sum of input_mask_expanded after clamping its values to a minimum of 1e-9. 39 | """ 40 | token_embeddings = model_output[0] 41 | input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() 42 | return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp( 43 | input_mask_expanded.sum(1), min=1e-9 44 | ) 45 | 46 | class DiversityRewardModel( BaseRewardModel ): 47 | 48 | diversity_model_path = "sentence-transformers/all-mpnet-base-v2" 49 | 50 | @property 51 | def name(self) -> str: return RewardModelType.diversity.value 52 | 53 | def __init__( self, device: str ): 54 | super().__init__() 55 | self.device = device 56 | self.tokenizer = AutoTokenizer.from_pretrained( DiversityRewardModel.diversity_model_path ) 57 | self.model = AutoModel.from_pretrained( DiversityRewardModel.diversity_model_path ).to(self.device) 58 | self.reward_bottom_k = 2 59 | self.history_reward_bottom_k = 2 60 | self.historic_embeddings = torch.tensor([]).to(self.device) 61 | self.history_range = (500, 15500) 62 | 63 | def get_embeddings( self, sentences: List[str] ) -> "torch.FloatTensor": 64 | """Runs a forward pass through the model. 65 | Args: 66 | sentences (:obj:`List[str]`): 67 | text message to be encoded. 68 | Returns: 69 | embedding (:obj:`torch.FloatTensor`): 70 | Embedding for the message. 71 | """ 72 | # Tokenizing sentences 73 | 74 | encoded_input = self.tokenizer( 75 | sentences, 76 | padding=True, 77 | truncation=True, 78 | return_tensors="pt", 79 | ).to(self.device) 80 | 81 | # Compute token embedding 82 | with torch.no_grad(): 83 | embeddings = self.model(**encoded_input) 84 | 85 | # Pooling 86 | sentence_embeddings = mean_pooling(embeddings, encoded_input["attention_mask"]) 87 | 88 | # Normalizing 89 | sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1) 90 | return sentence_embeddings 91 | 92 | def update_historic_embeddings( self, embeddings: torch.FloatTensor ): 93 | def unique(embeddings): 94 | unique_embeddings = [embeddings[0]] 95 | last_emb = embeddings[0] 96 | for emb in embeddings: 97 | if not torch.all(torch.eq(emb, last_emb)): 98 | unique_embeddings.append(emb) 99 | last_emb = emb 100 | return torch.stack(unique_embeddings) 101 | 102 | embeddings_unique = unique(embeddings) 103 | historic_embeddings = torch.cat([self.historic_embeddings, embeddings_unique]) 104 | self.historic_embeddings = historic_embeddings[-self.history_range[1]:, :] 105 | 106 | def get_historic_rewards( self, embeddings: torch.FloatTensor ) -> torch.FloatTensor: 107 | def regularise( rewards ): 108 | # sigmoid function that cutoff at 0.05 approximately 109 | return 1/(1 + torch.exp(-1000 * rewards + 50)) 110 | 111 | # Return None if history size is too small 112 | if self.historic_embeddings.shape[0] < (self.history_range[0] + self.history_reward_bottom_k): 113 | return None 114 | 115 | # Calculate the pairwise cosine similarity. 116 | similarity = pairwise_cosine_similarity( embeddings, self.historic_embeddings[self.history_range[0]:] ) 117 | 118 | # Reward to be at the bottom_k smallest of the 1 - similarity score. 119 | rewards = torch.topk((1 - torch.abs(similarity)), self.history_reward_bottom_k, largest = False)[0][:, -1] 120 | 121 | return regularise(rewards) 122 | 123 | def get_batch_rewards( self, embeddings: torch.FloatTensor ) -> torch.FloatTensor: 124 | def regularise( rewards ): 125 | # sigmoid function that maps 0.07 -> 0.23; 0.1 -> 0.5; 0.2 -> 0.98 126 | return 1/(1 + torch.exp(-40 * rewards + 4)) 127 | 128 | # Calculate the pairwise cosine similarity. 129 | similarity = pairwise_cosine_similarity( embeddings, embeddings ) 130 | 131 | # Reward to be at the 10% quantile of the 1 - similarity score. 132 | rewards = torch.topk((1 - torch.abs(similarity)), self.reward_bottom_k, largest = False)[0][:, -1] 133 | 134 | return regularise(rewards) 135 | 136 | def get_rewards( self, prompt: str, completions: List[str], name: str ) -> torch.FloatTensor: 137 | # Check if completions are empty, return 0 if so 138 | if len(completions) == 0: 139 | return torch.tensor([]).to(self.device) 140 | 141 | # Get embeddings for all completions. 142 | embeddings = self.get_embeddings( completions ) 143 | 144 | # Get batch rewards. 145 | batch_rewards = self.get_batch_rewards(embeddings) 146 | 147 | # get historic rewards. 148 | historic_rewards = self.get_historic_rewards(embeddings) 149 | 150 | self.update_historic_embeddings(embeddings) 151 | 152 | # Return all 153 | if historic_rewards != None: 154 | return batch_rewards * historic_rewards 155 | else: 156 | return batch_rewards 157 | 158 | def normalize_rewards( self, rewards: torch.FloatTensor ) -> torch.FloatTensor: 159 | return rewards -------------------------------------------------------------------------------- /openvalidators/reward/dpo.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | 18 | import torch 19 | import bittensor as bt 20 | from typing import List 21 | from .config import RewardModelType 22 | from .reward import BaseRewardModel 23 | from transformers import AutoTokenizer, AutoModelForCausalLM 24 | 25 | 26 | class DirectPreferenceRewardModel(BaseRewardModel): 27 | 28 | reward_model_name: str = "cerebras/btlm-3b-8k-base" 29 | 30 | @property 31 | def name(self) -> str: return RewardModelType.dpo.value 32 | 33 | def __init__(self, device: str): 34 | super().__init__() 35 | self.device = device 36 | self.penalty = 1.2 # Same penalty as the original [paper](https://arxiv.org/pdf/1909.05858.pdf). 37 | self.tokenizer = AutoTokenizer.from_pretrained(DirectPreferenceRewardModel.reward_model_name) 38 | self.model = AutoModelForCausalLM.from_pretrained(DirectPreferenceRewardModel.reward_model_name, 39 | trust_remote_code=True, 40 | torch_dtype=torch.float16).to(self.device) 41 | 42 | def reward_single(self, prompt: str, completion: str, name: str ,with_penalty=True) -> float: 43 | r""" Calculates a direct preference optimization (DPO) style reward for a completion, 44 | which is a reference model's average log-probability for completion tokens given a prompt. 45 | Uses guidance from https://github.com/eric-mitchell/direct-preference-optimization/blob/main/trainers.py. 46 | """ 47 | with torch.no_grad(): 48 | 49 | # Check if completion is 50 | if completion.strip() == '' or len(completion) <= 5: 51 | return -11 # exp(-11)=1.67e-5 < 2e-5=1/50257 (typical vocab size) 52 | 53 | # Tokenize the combined prompt + completion. 54 | combined = self.tokenizer(prompt + completion, return_tensors="pt").input_ids[0].to(self.device) # [seq_len] 55 | # Tokenize only the prompt, to help determine prompt token length. 56 | prompt_part = self.tokenizer(prompt, return_tensors="pt").input_ids[0].to(self.device) # [prompt_len] 57 | 58 | # Completion doesn't fit into model sequence, so return lowest reward. 59 | if self.tokenizer.model_max_length <= len(prompt_part): 60 | return -11. # exp(-11)=1.67e-5 < 2e-5=1/50257 (typical vocab size) 61 | 62 | # Truncate combined to fit into model max sequence length. 63 | if self.tokenizer.model_max_length < len(combined): 64 | combined = combined[:self.tokenizer.model_max_length] 65 | 66 | labels = combined.clone() # [seq_len] 67 | # Ignore prompt part for calculating reward. 68 | labels[:len(prompt_part)] = -100 69 | # Label only each next token prediction ground-truth. 70 | labels = labels[1:] # [seq_len-1] 71 | loss_mask = (labels != -100) # [seq_len-1] 72 | # Dummy token to allow for indexing, but loss will be ignored. 73 | labels[labels == -100] = 0 74 | # Reshape for gather operation. 75 | labels = labels.unsqueeze(0).unsqueeze(2) # [batch_size=1, seq_len-1, :] 76 | 77 | # Forward pass to calculate logit predictions for each sequence position. 78 | logits = self.model(combined.unsqueeze(0)).logits # [batch_size=1, seq_len, vocab_len] 79 | # Predict only where labels are available. 80 | logits = logits[:, :-1, :] # [batch_size=1, seq_len-1, vocab_len] 81 | 82 | if with_penalty: 83 | # Apply penalty for repeated generation 84 | for i in range(len(prompt_part)+1, len(combined)-1): 85 | logit = logits[:,i,:].clone() 86 | inputs = combined[len(prompt_part):i].clone() 87 | logits[:,i,:] = self.logit_penalty(input_ids=inputs, logit=logit) 88 | 89 | # Rescale via log(softmax(logits)). 90 | logits = logits.log_softmax(-1) 91 | # Calculate the model's log-probability for each actual completion token. 92 | per_token_logps = torch.gather(logits, dim=2, index=labels).squeeze(2) # [batch_size=1, seq_len-1] 93 | # Average log-probability over completion sequence. 94 | reward = (per_token_logps * loss_mask).sum(-1) / loss_mask.sum(-1) # [batch_size=1] 95 | reward = reward[0].cpu().detach() 96 | 97 | # NaNs can possibly arise through log(0)=-inf, replace with suitably small logits. 98 | if torch.isnan(reward) or torch.isinf(reward): 99 | return -11. # exp(-11)=1.67e-5 < 2e-5=1/50257 (typical vocab size) 100 | return reward.item() 101 | 102 | def get_rewards(self, prompt: str, completions: List[str], name: str) -> torch.FloatTensor: 103 | rewards = torch.tensor([self.reward_single(prompt, completion, name) for completion in completions], 104 | dtype=torch.float32).to(self.device) 105 | bt.logging.trace(f"DirectPreferenceRewardModel | rewards: {rewards.tolist()}") 106 | return rewards 107 | 108 | def logit_penalty(self, input_ids: torch.LongTensor, logit: torch.FloatTensor) -> torch.FloatTensor: 109 | # Counts the unique tokens within each generation 110 | uniques, counts = input_ids.unique(return_counts=True) 111 | score = torch.gather(logit, 1, uniques.unsqueeze(0)) 112 | 113 | # if score < 0 then repetition penalty has to be multiplied to reduce the previous token probability 114 | score = torch.where(score < 0, score * (self.penalty**counts), score / (self.penalty**counts)) 115 | 116 | logit.scatter_(1, uniques.unsqueeze(0), score.to(logit.dtype)) 117 | return logit -------------------------------------------------------------------------------- /openvalidators/reward/nsfw.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | # The MIT License (MIT) 4 | # Copyright © 2021 Yuma Rao 5 | 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 7 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 8 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 9 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 10 | 11 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 12 | # the Software. 13 | 14 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 15 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 16 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 17 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 18 | # DEALINGS IN THE SOFTWARE. 19 | 20 | import torch 21 | from typing import List 22 | from .config import RewardModelType 23 | from .reward import BaseRewardModel 24 | from transformers import AutoModelForSequenceClassification, AutoTokenizer 25 | 26 | class NSFWRewardModel( BaseRewardModel ): 27 | 28 | nsfw_filter_model_path = "facebook/roberta-hate-speech-dynabench-r4-target" 29 | 30 | @property 31 | def name(self) -> str: return RewardModelType.nsfw.value 32 | 33 | def __init__( self, device: str ): 34 | super().__init__() 35 | self.device = device 36 | self.tokenizer = AutoTokenizer.from_pretrained( NSFWRewardModel.nsfw_filter_model_path ) 37 | self.model = AutoModelForSequenceClassification.from_pretrained( NSFWRewardModel.nsfw_filter_model_path ).to(self.device) 38 | 39 | def reward( self, prompt: str, completion: str, name: str ) -> float: 40 | 41 | boundary = -0.5 42 | with torch.no_grad(): 43 | message = completion 44 | input_ids = self.tokenizer(message)["input_ids"] 45 | 46 | # Returns the nsfw hate score for the chunk. 47 | def hate_score( chunk ) -> float: 48 | nothate, hate = self.model(torch.tensor( [chunk] ).to(self.device)).logits[0].tolist() 49 | return nothate, hate 50 | 51 | # Returns the max hate score from each chunk of size chunk_size 52 | def sum_nsfw_scores(input_ids, chunk_size ): 53 | max_score = -1000 54 | for i in range(0, len(input_ids), chunk_size): 55 | chunk_i = input_ids[i:i + chunk_size] 56 | not_hate_score_i, hate_score_i = hate_score( chunk_i ) 57 | max_score = max( max( -not_hate_score_i, hate_score_i), max_score) 58 | return max_score 59 | 60 | # 0 when needs to be filtered out, 1 when it is safe 61 | return 0.0 if sum_nsfw_scores( input_ids, chunk_size = 512 ) > boundary else 1.0 62 | 63 | def get_rewards( self, prompt: str, completions: List[str], name: str ) -> torch.FloatTensor: 64 | return torch.tensor( [self.reward( prompt, completion, name ) for completion in completions], dtype=torch.float32).to(self.device) 65 | 66 | def normalize_rewards( self, rewards: torch.FloatTensor ) -> torch.FloatTensor: 67 | return rewards -------------------------------------------------------------------------------- /openvalidators/reward/open_assistant.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | 18 | import torch 19 | from typing import List 20 | from .config import RewardModelType 21 | from .reward import BaseRewardModel 22 | from transformers import AutoTokenizer, AutoModelForSequenceClassification 23 | 24 | class OpenAssistantRewardModel( BaseRewardModel ): 25 | 26 | reward_model_name: str = "OpenAssistant/reward-model-deberta-v3-large-v2" 27 | 28 | @property 29 | def name(self) -> str: return RewardModelType.rlhf.value 30 | 31 | def __init__( self , device: str ): 32 | super().__init__() 33 | self.device = device 34 | self.tokenizer = AutoTokenizer.from_pretrained( OpenAssistantRewardModel.reward_model_name ) 35 | self.model = AutoModelForSequenceClassification.from_pretrained( OpenAssistantRewardModel.reward_model_name ) .to(self.device) 36 | 37 | def reward_single( self, prompt: str, completion: str, name: str ) -> float: 38 | with torch.no_grad(): 39 | inputs = self.tokenizer(prompt, completion, return_tensors='pt').to(self.device) 40 | return float( self.model( **inputs ).logits[0].cpu().detach() ) 41 | 42 | def get_rewards( self, prompt: str, completions: List[str], name: str ) -> torch.FloatTensor: 43 | return torch.tensor( [self.reward_single( prompt, completion, name ) for completion in completions], dtype=torch.float32).to(self.device) 44 | -------------------------------------------------------------------------------- /openvalidators/reward/prompt.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | 18 | import time 19 | import torch 20 | import bittensor as bt 21 | from typing import List 22 | from .config import RewardModelType 23 | from .reward import BaseRewardModel 24 | from openvalidators.prompts import AugmentPrompt, FollowupPrompt, AnswerPrompt 25 | from transformers import AutoTokenizer, AutoModelForCausalLM 26 | 27 | 28 | class PromptRewardModel(BaseRewardModel): 29 | reward_model_name: str = "VMware/open-llama-7b-open-instruct" 30 | 31 | @property 32 | def name(self) -> str: return RewardModelType.prompt.value 33 | 34 | def __init__(self, device: str ): 35 | super().__init__() 36 | self.device = device 37 | 38 | # https://huggingface.co/VMware/open-llama-7b-open-instruct 39 | # Fast tokenizer results in incorrect encoding, set the use_fast = False parameter. 40 | self.tokenizer = AutoTokenizer.from_pretrained(PromptRewardModel.reward_model_name, use_fast=False) 41 | # Generative default expects most recent token on right-hand side with padding on left. 42 | # https://github.com/huggingface/transformers/pull/10552 43 | self.tokenizer.padding_side = "left" 44 | 45 | self.model = AutoModelForCausalLM.from_pretrained(PromptRewardModel.reward_model_name, 46 | torch_dtype=torch.float16).to(self.device) 47 | 48 | def reward(self, prompt: str, completion: str, name: str) -> float: 49 | with torch.no_grad(): 50 | # Choose correct scoring prompt for request type. 51 | if name == 'augment': 52 | scoring_prompt = AugmentPrompt() 53 | elif name == 'followup': 54 | scoring_prompt = FollowupPrompt() 55 | elif name == 'answer': 56 | scoring_prompt = AnswerPrompt() 57 | else: 58 | return 0 59 | 60 | # Format scoring prompt for this completion. 61 | scoring_prompt_text = scoring_prompt.text(prompt, completion) 62 | 63 | # Tokenize formatted scoring prompt. 64 | encodings_dict = self.tokenizer( 65 | scoring_prompt_text, 66 | truncation=False, 67 | max_length=2048, 68 | padding="max_length", 69 | return_tensors="pt", 70 | ) 71 | input_ids = encodings_dict["input_ids"].to(self.device) 72 | 73 | # Prompt local reward model. 74 | start_time = time.time() 75 | generated_tokens = self.model.generate(input_ids, max_new_tokens=2, max_time=1) 76 | duration = time.time() - start_time 77 | generated_text = self.tokenizer.batch_decode(generated_tokens, skip_special_tokens=True) 78 | 79 | # Extract score from generated text. 80 | score_text = generated_text[0][len(scoring_prompt_text):] 81 | score = scoring_prompt.extract_score(score_text) 82 | bt.logging.trace(f"PromptRewardModel | {name} score: {score} | {repr(score_text)} | " 83 | f"{duration:.2f}s | {repr(completion[:70])}") 84 | 85 | # Scale 0-10 score to 0-1 range. 86 | score /= 10. 87 | 88 | return score 89 | 90 | def get_rewards( self, prompt: str, completions: List[str], name: str ) -> torch.FloatTensor: 91 | bt.logging.debug(f"PromptRewardModel | Calculating {len(completions)} rewards (typically < 1 sec/reward).") 92 | bt.logging.trace(f"PromptRewardModel | prompt: {repr(prompt[:50])} ... {repr(prompt[-50:])}") 93 | return torch.tensor( [self.reward( prompt, completion, name ) for completion in completions], dtype=torch.float32).to(self.device) 94 | 95 | -------------------------------------------------------------------------------- /openvalidators/reward/reciprocate.py: -------------------------------------------------------------------------------- 1 | 2 | # The MIT License (MIT) 3 | # Copyright © 2021 Yuma Rao 4 | 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 6 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 7 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 8 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 9 | 10 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 11 | # the Software. 12 | 13 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 14 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 15 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 16 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 17 | # DEALINGS IN THE SOFTWARE. 18 | 19 | import torch 20 | from typing import List 21 | from .config import RewardModelType 22 | from .reward import BaseRewardModel 23 | from transformers import AutoTokenizer, AutoModelForSequenceClassification 24 | 25 | class ReciprocateRewardModel( BaseRewardModel ): 26 | 27 | reward_model_path: str = "reciprocate/gpt-j_rm_format-oa" 28 | revision: str = "501f895" 29 | 30 | @property 31 | def name(self) -> str: return RewardModelType.reciprocate.value 32 | 33 | def __init__( self, device: str ): 34 | super().__init__() 35 | self.device = device 36 | self.tokenizer = AutoTokenizer.from_pretrained( ReciprocateRewardModel.reward_model_path, revision = ReciprocateRewardModel.revision ) 37 | self.model = AutoModelForSequenceClassification.from_pretrained( ReciprocateRewardModel.reward_model_path, 38 | revision = ReciprocateRewardModel.revision, 39 | torch_dtype=torch.float16).to(self.device) 40 | 41 | def reward( self, prompt: str, completion: str, name: str ) -> float: 42 | with torch.no_grad(): 43 | message = f"<|prompter|>{prompt}<|assistant|>{completion}<|endoftext|>" 44 | inputs = self.tokenizer( message, 45 | return_tensors="pt" , 46 | truncation=True, 47 | ).to(self.device) 48 | return float( self.model( **inputs )[0].item() ) 49 | 50 | def get_rewards( self, prompt: str, completions: List[str], name: str ) -> torch.FloatTensor: 51 | return torch.tensor( [self.reward( prompt, completion, name ) for completion in completions], dtype=torch.float32).to(self.device) 52 | 53 | -------------------------------------------------------------------------------- /openvalidators/reward/relevance.py: -------------------------------------------------------------------------------- 1 | 2 | # The MIT License (MIT) 3 | # Copyright © 2021 Yuma Rao 4 | 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 6 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 7 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 8 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 9 | 10 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 11 | # the Software. 12 | 13 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 14 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 15 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 16 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 17 | # DEALINGS IN THE SOFTWARE. 18 | 19 | import torch 20 | from typing import List 21 | from .config import RewardModelType 22 | from .reward import BaseRewardModel 23 | from transformers import AutoTokenizer, AutoModel 24 | from torchmetrics.functional import pairwise_cosine_similarity 25 | import torch.nn.functional as F 26 | 27 | 28 | def mean_pooling(model_output, attention_mask): 29 | """Applies mean pooling to the token embeddings generated by the model. 30 | Args: 31 | model_output (torch.Tensor): Embedding model output, where the first element contains token embeddings. 32 | attention_mask (torch.Tensor): Attention mask to indicate valid tokens. 33 | Returns: 34 | torch.Tensor: Mean-pooled representation of the token embeddings. 35 | Notes: 36 | - The function calculates the mean-pooled representation using the attention mask for valid tokens. 37 | - Input_mask_expanded is created by expanding the attention mask to match the size of token embeddings. 38 | - The result is obtained by summing the element-wise multiplication of embeddings and input_mask_expanded, 39 | and dividing it by the sum of input_mask_expanded after clamping its values to a minimum of 1e-9. 40 | """ 41 | token_embeddings = model_output[0] 42 | input_mask_expanded = ( 43 | attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() 44 | ) 45 | return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp( 46 | input_mask_expanded.sum(1), min=1e-9 47 | ) 48 | 49 | class RelevanceRewardModel( BaseRewardModel ): 50 | 51 | @property 52 | def name(self) -> str: return RewardModelType.relevance.value 53 | 54 | def __init__( self, device: str ): 55 | super().__init__() 56 | self.device = device 57 | self.models = [ 58 | BertRelevanceRewardModel(self.device), 59 | MpnetRelevenceModel(self.device) 60 | ] 61 | self.bounds = [-0.0246, 0.3] 62 | 63 | def get_rewards( self, prompt: str, completions: List[str], name: str ) -> torch.FloatTensor: 64 | return torch.tensor( [self.reward( prompt, completion, name ) for completion in completions], dtype=torch.float32).to(self.device) 65 | 66 | def normalize_rewards( self, rewards: torch.FloatTensor ) -> torch.FloatTensor: 67 | return rewards 68 | 69 | def reward(self, prompt: str, completion: str, name: str) -> float: 70 | for i, model in enumerate(self.models): 71 | 72 | # rewards 73 | diff = model.reward(prompt,completion) 74 | 75 | # If a model returns 0, stop iterating and return 0 76 | if diff < self.bounds[i]: 77 | return 0.0 78 | # If none of the models returned 0, return 1 79 | return 1.0 80 | 81 | class BertRelevanceRewardModel( BaseRewardModel ): 82 | 83 | relevance_model_path = "bert-base-uncased" 84 | 85 | def __init__( self, device: str ): 86 | super().__init__() 87 | self.device = device 88 | self.tokenizer = AutoTokenizer.from_pretrained(BertRelevanceRewardModel.relevance_model_path) 89 | self.model = AutoModel.from_pretrained(BertRelevanceRewardModel.relevance_model_path).to(self.device) 90 | 91 | def get_embedding(self, message: str) -> "torch.FloatTensor": 92 | """Runs a forward pass through the model. 93 | Args: 94 | message (:obj:`str`): 95 | text message to be encoded. 96 | Returns: 97 | embedding (:obj:`torch.FloatTensor`): 98 | Embedding for the message. 99 | """ 100 | encoded_input = self.tokenizer( 101 | message, 102 | padding=True, 103 | truncation=True, 104 | return_overflowing_tokens=True, 105 | return_tensors="pt", 106 | ).to(self.device) 107 | 108 | # Pop the overflow mapping from the input to maintain the expected { input_ids, mask } format of the model 109 | _ = encoded_input.pop("overflow_to_sample_mapping") 110 | 111 | with torch.no_grad(): 112 | embeddings = self.model(**encoded_input) 113 | 114 | sentence_embeddings = mean_pooling(embeddings, encoded_input["attention_mask"]) 115 | sentence_embeddings = torch.nn.functional.normalize(sentence_embeddings, p=2, dim=1) 116 | batch_representation = torch.mean(sentence_embeddings, dim=0) 117 | return batch_representation 118 | 119 | def reward( self, prompt: str, completion:str ) -> float: 120 | # Get the two bert embeddings. 121 | completion_embedding = self.get_embedding( completion) 122 | prompt_embedding = self.get_embedding( prompt) 123 | 124 | # Calculate the RMSE distance for the 2 embeddings. 125 | diff = (( completion_embedding - prompt_embedding )**2).mean()**0.5 126 | 127 | # Return relevance scoring. 128 | return float(-diff) 129 | 130 | class MpnetRelevenceModel( BaseRewardModel ): 131 | 132 | diversity_model_path = "sentence-transformers/all-mpnet-base-v2" 133 | 134 | def __init__( self, device: str ): 135 | super().__init__() 136 | self.device = device 137 | self.tokenizer = AutoTokenizer.from_pretrained( MpnetRelevenceModel.diversity_model_path ) 138 | self.model = AutoModel.from_pretrained( MpnetRelevenceModel.diversity_model_path ).to(self.device) 139 | self.reward_quantile = torch.tensor(0.1).to(self.device) 140 | 141 | def get_embeddings( self, sentences: List[str] ) -> "torch.FloatTensor": 142 | """Runs a forward pass through the model. 143 | Args: 144 | sentences (:obj:`List[str]`): 145 | text message to be encoded. 146 | Returns: 147 | embedding (:obj:`torch.FloatTensor`): 148 | Embedding for the message. 149 | """ 150 | # Tokenizing sentences 151 | 152 | encoded_input = self.tokenizer( 153 | sentences, 154 | padding=True, 155 | truncation=True, 156 | return_tensors="pt", 157 | ).to(self.device) 158 | 159 | # Compute token embedding 160 | with torch.no_grad(): 161 | embeddings = self.model(**encoded_input) 162 | 163 | # Pooling 164 | sentence_embeddings = mean_pooling(embeddings, encoded_input["attention_mask"]) 165 | 166 | # Normalizing 167 | sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1) 168 | return sentence_embeddings 169 | 170 | def reward( self, prompt: str, completion: str ) -> torch.FloatTensor: 171 | 172 | # Get embeddings for all completions. 173 | embeddings = self.get_embeddings( completion ) 174 | prompt_embed = self.get_embeddings( prompt ) 175 | 176 | # Calculate the pairwise cosine similarity. 177 | similarity = pairwise_cosine_similarity( prompt_embed, embeddings ) 178 | 179 | return torch.abs(similarity) -------------------------------------------------------------------------------- /openvalidators/reward/reward.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | 18 | import torch 19 | import bittensor as bt 20 | from typing import List 21 | from abc import abstractmethod 22 | 23 | class BaseRewardModel: 24 | 25 | @property 26 | @abstractmethod 27 | def name(self) -> str: ... 28 | def __str__(self) -> str: return str(self.name) 29 | def __repr__(self) -> str: return str(self.name) 30 | 31 | @abstractmethod 32 | def get_rewards( self, prompt: str, completion: List[str], name: str ) -> torch.FloatTensor: ... 33 | 34 | def __init__(self) -> None: 35 | self.count = 0 36 | self.mean = 0.0 37 | self.var = 0.0 38 | self.count_limit = 3000 39 | 40 | def normalize_rewards( self, rewards: torch.FloatTensor ) -> torch.FloatTensor: 41 | """ 42 | This method normalizes the given rewards by updating the moving mean and variance statistics. The rewards are first standardized, and then scaled to the 0-1 range using a cumulative distribution function (CDF) to ensure they're in a comparable range across different environments. 43 | 44 | Args: 45 | rewards (torch.FloatTensor): The reward values to be normalized. 46 | 47 | Returns: 48 | torch.FloatTensor: The normalized reward values. 49 | 50 | Note: 51 | - This function uses Welford's online algorithm to update the mean and variance. 52 | - It standardizes the reward values using the updated mean and variance. 53 | - It then scales the standardized values to the 0-1 range using the error function (erf) as a CDF. 54 | """ 55 | # Get the number of rewards (successful responses). 56 | new_count = rewards.numel() 57 | 58 | # Update stats only if there are new rewards. 59 | if 0 < new_count and 0 < self.count + new_count: 60 | # Calculate the mean and standard deviation of the new rewards. 61 | new_mean = rewards.mean() 62 | new_var = rewards.var(dim=0) 63 | 64 | # Compute the weights for the new and old rewards. 65 | new_weight = new_count / (self.count + new_count) 66 | old_weight = self.count / (self.count + new_count) 67 | 68 | # Save the difference in means before updating the old mean. 69 | diff = new_mean - self.mean 70 | 71 | # Update the old mean with the new mean and weights. 72 | self.mean = new_weight * new_mean + old_weight * self.mean 73 | # Update the old variance with the new variance and weights, and adjusting for the difference in means. 74 | self.var = (new_weight * new_var) + (old_weight * self.var) + (new_weight * old_weight) * diff * diff 75 | # Update the old count with the new count, but don't exceed the limit. 76 | self.count = min(self.count_limit, self.count + new_count) 77 | 78 | # Standardize the rewards using the updated mean and variance. 79 | rewards = rewards - self.mean 80 | if self.var > 0: 81 | rewards /= torch.sqrt(self.var) 82 | # Scale the standardized rewards to the range [0, 1] using the error function as a cumulative distribution function (CDF). 83 | rewards = 0.5 * (1 + torch.erf(rewards / torch.sqrt(torch.tensor([2.0])).to(rewards.device))) 84 | 85 | return rewards 86 | 87 | def apply( self, prompt: str, responses: List[ bt.DendriteCall ], name: str) -> torch.FloatTensor: 88 | """ Applies the reward model across each call. Unsuccessful responses are zeroed. 89 | """ 90 | # Get indices of correctly responding calls. 91 | 92 | successful_completions_indices: List[int] = [ idx for idx, resp in enumerate(responses) if resp.is_success ] 93 | 94 | # Get all completions from responding calls. 95 | successful_completions: List[str] = [ responses[idx].completion.strip() for idx in successful_completions_indices] 96 | 97 | # Reward each completion. 98 | successful_rewards = self.get_rewards( prompt, successful_completions, name ) 99 | 100 | # Softmax rewards across samples. 101 | successful_rewards_normalized = self.normalize_rewards( successful_rewards ) 102 | 103 | # Init zero rewards for all calls. 104 | filled_rewards = torch.ones( len( responses ), dtype=torch.float32) * torch.nan 105 | filled_rewards_normalized = torch.zeros( len( responses ), dtype=torch.float32) 106 | 107 | # Fill reward tensor. 108 | for idx, reward, reward_normalized in zip(successful_completions_indices, successful_rewards, successful_rewards_normalized): 109 | filled_rewards[idx] = reward 110 | filled_rewards_normalized[idx] = reward_normalized 111 | 112 | # Return the filled rewards. 113 | return filled_rewards, filled_rewards_normalized 114 | 115 | 116 | class MockRewardModel( BaseRewardModel ): 117 | 118 | @property 119 | def name(self) -> str: return self.mock_name 120 | 121 | def __init__(self, mock_name: str = 'MockReward'): 122 | super().__init__() 123 | self.mock_name = mock_name 124 | 125 | def apply( self, prompt: str, completion: List[str], name: str ) -> torch.FloatTensor: 126 | mock_reward = torch.tensor( [0 for _ in completion], dtype=torch.float32 ) 127 | return mock_reward, mock_reward 128 | 129 | -------------------------------------------------------------------------------- /openvalidators/reward/task_validator.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | import torch 18 | from typing import List 19 | from .config import RewardModelType 20 | from .reward import BaseRewardModel 21 | 22 | 23 | class TaskValidator( BaseRewardModel ): 24 | 25 | @property 26 | def name(self) -> str: return RewardModelType.task_validator.value 27 | 28 | def __init__(self): 29 | super().__init__() 30 | 31 | def reward( self, prompt: str, completion: str, name: str ) -> float: 32 | summary_keywords = ['Summary:', 'Paraphrase:', 'Paraphrasing:', 'Paraphrased:'] 33 | question_keywords = ['Question:', 'Query:', 'Q:'] 34 | answer_keywords = ['Answer:', 'Response:', 'A:', 'Completion:'] 35 | 36 | completion_contains_answer = any(answer_keyword.lower() in completion.lower() for answer_keyword in answer_keywords) 37 | completion_contains_question = any(question_keyword.lower() in completion.lower() for question_keyword in question_keywords) 38 | completion_contains_summary = any(summary_keyword.lower() in completion.lower() for summary_keyword in summary_keywords) 39 | 40 | is_summarization_prompt = name == 'augment' 41 | is_question_prompt = name.startswith('followup') 42 | is_answer_prompt = name.startswith('answer') 43 | 44 | if (is_summarization_prompt or is_question_prompt) and completion_contains_answer: 45 | return 0.0 46 | 47 | if (is_summarization_prompt or is_answer_prompt) and completion_contains_question: 48 | return 0.0 49 | 50 | if not is_summarization_prompt and completion_contains_summary: 51 | return 0.0 52 | 53 | return 1 54 | 55 | def get_rewards( self, prompt: str, completions: List[str], name: str ) -> torch.FloatTensor: 56 | return torch.tensor( [self.reward( prompt, completion, name ) for completion in completions], dtype=torch.float32) 57 | 58 | def normalize_rewards( self, rewards: torch.FloatTensor ) -> torch.FloatTensor: 59 | return rewards 60 | 61 | def reset(self): 62 | pass 63 | 64 | -------------------------------------------------------------------------------- /openvalidators/run.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | 18 | import asyncio 19 | import bittensor as bt 20 | from traceback import print_exception 21 | 22 | from openvalidators.forward import forward 23 | from openvalidators.utils import should_checkpoint, checkpoint, should_reinit_wandb, reinit_wandb, load_state, save_state 24 | from openvalidators.weights import should_set_weights, set_weights 25 | from openvalidators.misc import ttl_get_block 26 | 27 | # Neuron run loop.` 28 | def run(self): 29 | bt.logging.info("run()") 30 | load_state(self) 31 | checkpoint(self) 32 | try: 33 | while True: 34 | bt.logging.info(f"step({self.step}) block({ttl_get_block( self )})") 35 | 36 | # Run multiple forwards. 37 | async def run_forward(): 38 | coroutines = [forward(self) for _ in range(self.config.neuron.num_concurrent_forwards)] 39 | await asyncio.gather(*coroutines) 40 | 41 | self.loop.run_until_complete(run_forward()) 42 | 43 | # Resync the network state 44 | if should_checkpoint(self): 45 | checkpoint(self) 46 | 47 | # Set the weights on chain. 48 | if should_set_weights(self): 49 | set_weights(self) 50 | save_state(self) 51 | 52 | # Rollover wandb to a new run. 53 | if should_reinit_wandb(self): 54 | reinit_wandb(self) 55 | 56 | self.prev_block = ttl_get_block(self) 57 | self.step += 1 58 | 59 | except Exception as e: 60 | bt.logging.error("Error in training loop", str(e)) 61 | bt.logging.debug(print_exception(value=e)) 62 | -------------------------------------------------------------------------------- /openvalidators/weights.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | 18 | # Utils for weights setting on chain. 19 | 20 | import wandb 21 | import torch 22 | import bittensor as bt 23 | from openvalidators.misc import ttl_get_block 24 | import openvalidators 25 | 26 | 27 | def should_set_weights(self) -> bool: 28 | # Check if enough epoch blocks have elapsed since the last epoch. 29 | if self.config.neuron.disable_set_weights: 30 | return False 31 | 32 | return ttl_get_block(self) % self.config.neuron.epoch_length < self.prev_block % self.config.neuron.epoch_length 33 | 34 | 35 | def set_weights(self): 36 | # Calculate the average reward for each uid across non-zero values. 37 | # Replace any NaN values with 0. 38 | raw_weights = torch.nn.functional.normalize(self.moving_averaged_scores, p=1, dim=0) 39 | bt.logging.trace("raw_weights", raw_weights) 40 | bt.logging.trace("top10 values", raw_weights.sort()[0]) 41 | bt.logging.trace("top10 uids", raw_weights.sort()[1]) 42 | 43 | # Process the raw weights to final_weights via subtensor limitations. 44 | (processed_weight_uids, processed_weights,) = bt.utils.weight_utils.process_weights_for_netuid( 45 | uids=self.metagraph.uids.to("cpu"), 46 | weights=raw_weights.to("cpu"), 47 | netuid=self.config.netuid, 48 | subtensor=self.subtensor, 49 | metagraph=self.metagraph, 50 | ) 51 | bt.logging.trace("processed_weights", processed_weights) 52 | bt.logging.trace("processed_weight_uids", processed_weight_uids) 53 | 54 | # Set the weights on chain via our subtensor connection. 55 | self.subtensor.set_weights( 56 | wallet=self.wallet, 57 | netuid=self.config.netuid, 58 | uids=processed_weight_uids, 59 | weights=processed_weights, 60 | wait_for_finalization=False, 61 | version_key=openvalidators.__spec_version__, 62 | ) 63 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | pythonpath = . -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | bittensor>=5.2.1,<6.0.0 2 | transformers==4.28.0 3 | wandb==0.15.3 4 | datasets==2.14.0 5 | plotly==5.14.1 6 | networkx==3.1 7 | scipy==1.10.1 8 | pre-commit==3.3.2 9 | click==8.1.3 10 | torchmetrics 11 | sentencepiece 12 | numpy==1.21.6 13 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Initialize variables 4 | script="openvalidators/neuron.py" 5 | autoRunLoc=$(readlink -f "$0") 6 | proc_name="openvalidators_main_process" 7 | args=() 8 | version_location="./openvalidators/__init__.py" 9 | version="__version__" 10 | 11 | old_args=$@ 12 | 13 | # Check if pm2 is installed 14 | if ! command -v pm2 &> /dev/null 15 | then 16 | echo "pm2 could not be found. To install see: https://pm2.keymetrics.io/docs/usage/quick-start/" 17 | exit 1 18 | fi 19 | 20 | # Checks if $1 is smaller than $2 21 | # If $1 is smaller than or equal to $2, then true. 22 | # else false. 23 | version_less_than_or_equal() { 24 | [ "$1" = "`echo -e "$1\n$2" | sort -V | head -n1`" ] 25 | } 26 | 27 | # Checks if $1 is smaller than $2 28 | # If $1 is smaller than $2, then true. 29 | # else false. 30 | version_less_than() { 31 | [ "$1" = "$2" ] && return 1 || version_less_than_or_equal $1 $2 32 | } 33 | 34 | # Returns the difference between 35 | # two versions as a numerical value. 36 | get_version_difference() { 37 | local tag1="$1" 38 | local tag2="$2" 39 | 40 | # Extract the version numbers from the tags 41 | local version1=$(echo "$tag1" | sed 's/v//') 42 | local version2=$(echo "$tag2" | sed 's/v//') 43 | 44 | # Split the version numbers into an array 45 | IFS='.' read -ra version1_arr <<< "$version1" 46 | IFS='.' read -ra version2_arr <<< "$version2" 47 | 48 | # Calculate the numerical difference 49 | local diff=0 50 | for i in "${!version1_arr[@]}"; do 51 | local num1=${version1_arr[$i]} 52 | local num2=${version2_arr[$i]} 53 | 54 | # Compare the numbers and update the difference 55 | if (( num1 > num2 )); then 56 | diff=$((diff + num1 - num2)) 57 | elif (( num1 < num2 )); then 58 | diff=$((diff + num2 - num1)) 59 | fi 60 | done 61 | 62 | strip_quotes $diff 63 | } 64 | 65 | read_version_value() { 66 | # Read each line in the file 67 | while IFS= read -r line; do 68 | # Check if the line contains the variable name 69 | if [[ "$line" == *"$version"* ]]; then 70 | # Extract the value of the variable 71 | local value=$(echo "$line" | awk -F '=' '{print $2}' | tr -d ' ') 72 | strip_quotes $value 73 | return 0 74 | fi 75 | done < "$version_location" 76 | 77 | echo "" 78 | } 79 | 80 | check_package_installed() { 81 | local package_name="$1" 82 | os_name=$(uname -s) 83 | 84 | if [[ "$os_name" == "Linux" ]]; then 85 | # Use dpkg-query to check if the package is installed 86 | if dpkg-query -W -f='${Status}' "$package_name" 2>/dev/null | grep -q "installed"; then 87 | return 1 88 | else 89 | return 0 90 | fi 91 | elif [[ "$os_name" == "Darwin" ]]; then 92 | if brew list --formula | grep -q "^$package_name$"; then 93 | return 1 94 | else 95 | return 0 96 | fi 97 | else 98 | echo "Unknown operating system" 99 | return 0 100 | fi 101 | } 102 | 103 | check_variable_value_on_github() { 104 | local repo="$1" 105 | local file_path="$2" 106 | local variable_name="$3" 107 | 108 | local url="https://api.github.com/repos/$repo/contents/$file_path" 109 | local response=$(curl -s "$url") 110 | 111 | # Check if the response contains an error message 112 | if [[ $response =~ "message" ]]; then 113 | echo "Error: Failed to retrieve file contents from GitHub." 114 | return 1 115 | fi 116 | 117 | # Extract the content from the response 118 | local content=$(echo "$response" | tr -d '\n' | jq -r '.content') 119 | 120 | if [[ "$content" == "null" ]]; then 121 | echo "File '$file_path' not found in the repository." 122 | return 1 123 | fi 124 | 125 | # Decode the Base64-encoded content 126 | local decoded_content=$(echo "$content" | base64 --decode) 127 | 128 | # Extract the variable value from the content 129 | local variable_value=$(echo "$decoded_content" | grep "$variable_name" | awk -F '=' '{print $2}' | tr -d ' ') 130 | 131 | if [[ -z "$variable_value" ]]; then 132 | echo "Variable '$variable_name' not found in the file '$file_path'." 133 | return 1 134 | fi 135 | 136 | strip_quotes $variable_value 137 | } 138 | 139 | strip_quotes() { 140 | local input="$1" 141 | 142 | # Remove leading and trailing quotes using parameter expansion 143 | local stripped="${input#\"}" 144 | stripped="${stripped%\"}" 145 | 146 | echo "$stripped" 147 | } 148 | 149 | # Loop through all command line arguments 150 | while [[ $# -gt 0 ]]; do 151 | arg="$1" 152 | 153 | # Check if the argument starts with a hyphen (flag) 154 | if [[ "$arg" == -* ]]; then 155 | # Check if the argument has a value 156 | if [[ $# -gt 1 && "$2" != -* ]]; then 157 | if [[ "$arg" == "--script" ]]; then 158 | script="$2"; 159 | shift 2 160 | else 161 | # Add '=' sign between flag and value 162 | args+=("'$arg'"); 163 | args+=("'$2'"); 164 | shift 2 165 | fi 166 | else 167 | # Add '=True' for flags with no value 168 | args+=("'$arg'"); 169 | shift 170 | fi 171 | else 172 | # Argument is not a flag, add it as it is 173 | args+=("'$arg '"); 174 | shift 175 | fi 176 | done 177 | 178 | # Check if script argument was provided 179 | if [[ -z "$script" ]]; then 180 | echo "The --script argument is required." 181 | exit 1 182 | fi 183 | 184 | branch=$(git branch --show-current) # get current branch. 185 | echo watching branch: $branch 186 | echo pm2 process name: $proc_name 187 | 188 | # Get the current version locally. 189 | current_version=$(read_version_value) 190 | 191 | # Check if script is already running with pm2 192 | if pm2 status | grep -q $proc_name; then 193 | echo "The script is already running with pm2. Stopping and restarting..." 194 | pm2 delete $proc_name 195 | fi 196 | 197 | # Run the Python script with the arguments using pm2 198 | echo "Running $script with the following pm2 config:" 199 | 200 | # Join the arguments with commas using printf 201 | joined_args=$(printf "%s," "${args[@]}") 202 | 203 | # Remove the trailing comma 204 | joined_args=${joined_args%,} 205 | 206 | # Create the pm2 config file 207 | echo "module.exports = { 208 | apps : [{ 209 | name : '$proc_name', 210 | script : '$script', 211 | interpreter: 'python3', 212 | min_uptime: '5m', 213 | max_restarts: '5', 214 | args: [$joined_args] 215 | }] 216 | }" > app.config.js 217 | 218 | # Print configuration to be used 219 | cat app.config.js 220 | 221 | pm2 start app.config.js 222 | 223 | # Check if packages are installed. 224 | check_package_installed "jq" 225 | if [ "$?" -eq 1 ]; then 226 | while true; do 227 | 228 | # First ensure that this is a git installation 229 | if [ -d "./.git" ]; then 230 | 231 | # check value on github remotely 232 | latest_version=$(check_variable_value_on_github "opentensor/validators" "openvalidators/__init__.py" "__version__ ") 233 | 234 | # If the file has been updated 235 | if version_less_than $current_version $latest_version; then 236 | echo "latest version $latest_version" 237 | echo "current version $current_version" 238 | diff=$(get_version_difference $latest_version $current_version) 239 | if [ "$diff" -eq 1 ]; then 240 | echo "current validator version:" "$current_version" 241 | echo "latest validator version:" "$latest_version" 242 | 243 | # Pull latest changes 244 | # Failed git pull will return a non-zero output 245 | if git pull origin $branch; then 246 | # latest_version is newer than current_version, should download and reinstall. 247 | echo "New version published. Updating the local copy." 248 | 249 | # Install latest changes just in case. 250 | pip install -e . 251 | 252 | # # Run the Python script with the arguments using pm2 253 | # TODO (shib): Remove this pm2 del in the next spec version update. 254 | pm2 del auto_run_validator 255 | echo "Restarting PM2 process" 256 | pm2 restart $proc_name 257 | 258 | # Update current version: 259 | current_version=$(read_version_value) 260 | echo "" 261 | 262 | # Restart autorun script 263 | echo "Restarting script..." 264 | ./$(basename $0) $old_args && exit 265 | else 266 | echo "**Will not update**" 267 | echo "It appears you have made changes on your local copy. Please stash your changes using git stash." 268 | fi 269 | else 270 | # current version is newer than the latest on git. This is likely a local copy, so do nothing. 271 | echo "**Will not update**" 272 | echo "The local version is $diff versions behind. Please manually update to the latest version and re-run this script." 273 | fi 274 | else 275 | echo "**Skipping update **" 276 | echo "$current_version is the same as or more than $latest_version. You are likely running locally." 277 | fi 278 | else 279 | echo "The installation does not appear to be done through Git. Please install from source at https://github.com/opentensor/validators and rerun this script." 280 | fi 281 | 282 | # Wait about 30 minutes 283 | # This should be plenty of time for validators to catch up 284 | # and should prevent any rate limitations by GitHub. 285 | sleep 1800 286 | done 287 | else 288 | echo "Missing package 'jq'. Please install it for your system first." 289 | fi 290 | -------------------------------------------------------------------------------- /scratch.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opentensor/validators/9e2172f3889faa9ae5bc7dbae08ba4291610a022/scratch.ipynb -------------------------------------------------------------------------------- /scripts/Makefile: -------------------------------------------------------------------------------- 1 | openvalidators_dataset: 2 | # Downloads all the runs from the project and exports them to a csv file 3 | python3 data_collector.py \ 4 | --download_all \ 5 | --export_path='openvalidators_dataset.csv' \ 6 | 7 | run_id_dataset: 8 | # Downloads a specific run from the project and exports it to a csv file 9 | python3 data_collector.py \ 10 | --export_path='$(RUN_ID)_openvalidators_dataset.csv' \ 11 | --wandb_run_id=$(RUN_ID) \ 12 | 13 | mining_dataset: 14 | # Downloads all the runs from the project with a mining dataset 15 | python3 data_collector.py \ 16 | --download_all \ 17 | --export_path='openvalidators_dataset.csv' \ 18 | --export_mining_dataset \ 19 | 20 | scored_mining_dataset: 21 | # Downloads all the runs from the project with a scored mining dataset 22 | python3 data_collector.py \ 23 | --download_all \ 24 | --export_path='openvalidators_dataset.csv' \ 25 | --export_mining_with_scoring_dataset \ 26 | 27 | openai_mining_dataset: 28 | python3 data_collector.py \ 29 | --download_all \ 30 | --export_path='openvalidators_dataset.csv' \ 31 | --export_openai_dataset \ 32 | -------------------------------------------------------------------------------- /scripts/README.md: -------------------------------------------------------------------------------- 1 | # Openvalidators wandb data collector 2 | 3 | This folder contain a set of scripts to automate the data collection of the openvalidators project in wandb. 4 | 5 | ## Requirements 6 | Ensure that the [requirements](../requirements.txt) of the project are properly installed on your machine. 7 | 8 | ```bash 9 | pip install -e ../setup.py # Install openvalidators from root folder 10 | ``` 11 | 12 | The current implementation requires you to be logged in wandb. You can do so by running the following command: 13 | ```bash 14 | wandb login 15 | ``` 16 | 17 | You will also need to install `make` to profit from the predefined targets in the [Makefile](Makefile). 18 | To install `make` on Ubuntu, run the following command: 19 | ```bash 20 | sudo apt install make 21 | ``` 22 | 23 | If you are on Mac, you can install it by running: 24 | ```bash 25 | brew install make 26 | ``` 27 | 28 | 29 | **Note:** All the data collection scripts are designed to be run from the `scripts` directory, 30 | so ensure that you are in the right place before starting: 31 | ```bash 32 | cd scripts # From root folder 33 | ``` 34 | 35 | ## Usage 36 | This repository provides a convenient way to collect data from the 37 | [WandB Openvalidators platform](https://wandb.ai/opentensor-dev/openvalidators) using the [data_collector.py](data_collector.py) 38 | tool. The repository includes a [Makefile](Makefile) that simplifies the execution of commands for data extraction by providing 39 | predefined targets. 40 | 41 | The [data_collector.py](data_collector.py) script is designed to extract data from WandB runs based on a given set of 42 | parameters. It supports various options to specify the type of data to collect and the configuration for exporting the 43 | data. 44 | 45 | The repository's Makefile includes the following targets to facilitate data collection: 46 | 47 | 48 | ### `openvalidators_dataset` 49 | 50 | ```bash 51 | make openvalidators_dataset 52 | ``` 53 | 54 | This command downloads all the runs from the latest version of the project and exports them to a CSV file named 55 | **openvalidators_dataset.csv**. 56 | It utilizes the following options under the hood of the [Makefile](Makefile): 57 | 58 | - `--download_all`: Downloads all the runs. 59 | - `--export_path`: Specifies the path and filename for the exported CSV file. 60 | 61 | --- 62 | 63 | ### `run_id_dataset` 64 | ```bash 65 | make run_id_dataset RUN_ID= 66 | ``` 67 | This command downloads a specific run from the project and exports it to a CSV file named 68 | **$(RUN_ID)_openvalidators_dataset.csv**. It utilizes the following options: 69 | 70 | - `--export_path`: Specifies the path and filename for the exported CSV file. 71 | - `--wandb_run_id`: Specifies the ID of the run to download. 72 | 73 | --- 74 | 75 | ### `mining_dataset` 76 | ```bash 77 | make mining_dataset 78 | ``` 79 | This command downloads all the runs from the latest version of the project with a mining dataset and exports them to a 80 | CSV file named **openvalidators_dataset.csv**. It utilizes the following options: 81 | 82 | - `--download_all`: Downloads all the runs. 83 | - `--export_path`: Specifies the path and filename for the exported CSV file. 84 | - `--export_mining_dataset`: Enables the export of mining dataset. 85 | 86 | --- 87 | 88 | ### `scored_mining_dataset` 89 | ```bash 90 | make scored_mining_dataset 91 | ``` 92 | 93 | This command downloads all the runs from the latest version of the project with a scored mining dataset and exports them 94 | to a CSV file named **openvalidators_dataset.csv**. It utilizes the following options: 95 | 96 | - `--download_all`: Downloads all the runs. 97 | - `--export_path`: Specifies the path and filename for the exported CSV file. 98 | - `--export_mining_with_scoring_dataset`: Enables the export of mining dataset with scoring. 99 | 100 | --- 101 | 102 | ### `openai_mining_dataset` 103 | ```bash 104 | make openai_mining_dataset 105 | ``` 106 | 107 | This command downloads all the runs from the latest version of the project and exports them to jsonl file named 108 | **openai_mining_dataset_openvalidators.jsonl** in the [openai fine-tuning format](https://platform.openai.com/docs/guides/fine-tuning/prepare-training-data). 109 | 110 | Note: Feel completely free to adjust the [data_collector.py](data_collector.py) script and [Makefile](Makefile) as necessary to 111 | match your project configuration and requirements. 112 | 113 | --- 114 | 115 | # Data collector parameters 116 | 117 | The data_collector.py script accepts the following parameters: 118 | 119 | - **--download_all**: This parameter is a flag that, when set, instructs the script to download all the runs 120 | **from the latest version of openvalidators.** 121 | By default, it is set to False. Example usage: `--download_all`. 122 | - **--include_tags**: This parameter allows you to specify a list of wandb tags to filter the data collection. 123 | Example usage: `--include_tags=0.1.0,nfsw_filter,your-hot-key`. 124 | - **--wandb_run_id**: This parameter allows you to specify the WandB run ID to download. It is used when you want to 125 | extract data from a specific run. By default, it is set to None. Example usage: `--wandb_run_id=ugutvtox`. 126 | > **Note:** The run_id can be retrieved at the end of your URL, e.g. : 127 | > - URL: https://wandb.ai/opentensor-dev/openvalidators/runs/ugutvtox 128 | > - Run_id: **ugutvtox**. 129 | - **--export_mining_dataset**: This parameter is a flag that, when set, enables the export of the mining dataset. 130 | It is used when you want to extract data specifically for mining purposes, in the following format: 131 | ```json 132 | { 133 | "base_prompt" : "best_followup", 134 | "answer_prompt" : "best_answer", 135 | //... 136 | } 137 | ``` 138 | By default, it is set to False. 139 | Example usage: `--export_mining_dataset`. 140 | - **--export_mining_with_scoring_dataset**: This parameter is a flag that, when set, enables the export of the mining 141 | dataset with scores, in the following format: 142 | ```json 143 | { 144 | "base_prompt" : { "best_followup" : "score" }, 145 | "answer_prompt" : { "best_answer" : "score" }, 146 | //... 147 | } 148 | ``` 149 | It is used when you want to extract data for mining with scoring. By default, it is set to False. 150 | Example usage: `--export_mining_with_scoring_dataset`. 151 | 152 | - **--export_path**: This parameter allows you to specify the path where the exported dataset will be saved. 153 | By default, it is set to **"validator_dataset.csv"**. Example usage: `--export_path=your_path.csv`. 154 | - **--mining_dataset_output_format:** This parameter allows you to specify the output format of the mining dataset. 155 | Defaults to `json`, currently supports `json` and `csv`. Example usage: `--mining_dataset_output_format=csv`. 156 | - **--blacklist_path**: This parameter allows you to specify the path to a file containing blacklist phrases. 157 | The script will exclude any data that contains these phrases. By default, it is set to [blacklist_phrases.txt](blacklist_phrases.txt). 158 | Example usage: `--blacklist_path=blacklist_phrases.txt`. 159 | - **--export_openai_dataset**: This parameter is a flag that, when set, enables the export of the mining dataset 160 | in the [jsonl openai format for fine-tuning](https://platform.openai.com/docs/guides/fine-tuning): 161 | ```json lines 162 | {"prompt": "base_prompt", "completion": "best_followup"}, 163 | {"prompt": "answer_prompt", "completion": "best_answer" } 164 | ... 165 | ``` 166 | 167 | Make sure to adjust the parameters accordingly when executing the [data_collector.py](data_collector.py) script for your 168 | specific data collection needs. -------------------------------------------------------------------------------- /scripts/blacklist_phrases.txt: -------------------------------------------------------------------------------- 1 | This is a complex issue that has many aspects to consider. 2 | [] -------------------------------------------------------------------------------- /scripts/data_collector.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | 18 | import bittensor as bt 19 | import argparse 20 | import json 21 | import pandas as pd 22 | import openvalidators 23 | import os 24 | from analysis.utils import get_runs, download_data 25 | from traceback import print_exception 26 | from typing import List 27 | from data_formatter import create_json_dataset, create_csv_dataset, create_openai_dataset 28 | 29 | 30 | DEFAULT_PROJECT = 'opentensor-dev/openvalidators' 31 | DEFAULT_FILTERS = {"tags": {"$in": [openvalidators.__version__]}} 32 | 33 | 34 | def read_file_into_array(file_path: str) -> List[str]: 35 | """Reads a file into an array of strings""" 36 | bt.logging.info(f"Loading blacklists phrases from {file_path}") 37 | with open(file_path, 'r') as file: 38 | lines = file.readlines() 39 | return [line.strip() for line in lines] 40 | 41 | 42 | def collect_data( 43 | download_all: bool, 44 | export_path: str, 45 | wandb_run_id: str = None, 46 | include_flags: str = None 47 | ) -> pd.DataFrame: 48 | """Collects data from wandb run logs and exports it to a csv file. 49 | Args: 50 | download_all (bool): Whether to download all the data or just a specific run. 51 | export_path (str): Path to export the data. 52 | wandb_run_id (str, optional): Wandb run id to download. Defaults to None. 53 | Returns: 54 | pd.DataFrame: Dataframe of wandb run logs. 55 | """ 56 | if download_all: 57 | if include_flags is not None: 58 | flags = include_flags.split(',') 59 | DEFAULT_FILTERS['tags']['$in'].extend(flags) 60 | 61 | bt.logging.info(f'Downloading all files with tags: {DEFAULT_FILTERS}') 62 | runs = get_runs(DEFAULT_PROJECT, filters=DEFAULT_FILTERS, return_paths=True) 63 | df = download_data(runs) 64 | df.to_csv(export_path) 65 | else: 66 | if wandb_run_id is None: 67 | raise Exception("Please specify a wandb run id to download") 68 | else: 69 | bt.logging.info(f'Downloading data from run id: {wandb_run_id}') 70 | df = download_data(f'{DEFAULT_PROJECT}/{wandb_run_id}') 71 | df.to_csv(export_path) 72 | 73 | bt.logging.info(f'Data collected successfully at: {export_path}') 74 | return df 75 | 76 | 77 | def create_mining_dataset( 78 | df: pd.DataFrame, 79 | export_path: str, 80 | mining_dataset_output_format: str, 81 | blacklist_phrases: List[str], 82 | with_score: bool =False, 83 | export_openai_dataset: bool = False): 84 | """Creates a dataset for mining from the dataframe of wandb run logs. 85 | Args: 86 | df (pd.DataFrame): Dataframe of wandb run logs. 87 | export_path (str): Path to export the dataset. 88 | with_score (bool, optional): Whether to include the score in the dataset. Defaults to False. 89 | """ 90 | filename, file_extension = os.path.splitext(export_path) 91 | mining_dataset_path = f'mining_dataset_{filename}.{mining_dataset_output_format}' 92 | 93 | if with_score: 94 | mining_dataset_path = f'scored_{mining_dataset_path}' 95 | 96 | bt.logging.info(f"Creating mining dataset: {mining_dataset_path}") 97 | 98 | if export_openai_dataset: 99 | jsonl_dataset = create_openai_dataset(df=df, blacklist=blacklist_phrases) 100 | 101 | with open("openai_mining_dataset_openvalidators.jsonl", "w") as file: 102 | file.write(jsonl_dataset) 103 | 104 | elif mining_dataset_output_format == 'json': 105 | dict_dataset = create_json_dataset( 106 | df=df, 107 | include_scoring=with_score, 108 | blacklist=blacklist_phrases, 109 | ) 110 | with open(mining_dataset_path, 'w') as json_file: 111 | json.dump(dict_dataset, json_file) 112 | 113 | elif mining_dataset_output_format == 'csv': 114 | df_dataset = create_csv_dataset( 115 | df=df, 116 | include_scoring=with_score, 117 | blacklist=blacklist_phrases, 118 | ) 119 | df_dataset.to_csv(mining_dataset_path) 120 | else: 121 | raise Exception(f"Invalid mining dataset output format: {mining_dataset_output_format}") 122 | 123 | bt.logging.info(f"Mining dataset exported successfully to {mining_dataset_path}") 124 | 125 | 126 | if __name__ == '__main__': 127 | try: 128 | # Create an ArgumentParser object 129 | parser = argparse.ArgumentParser() 130 | 131 | # Add the flags as parameters 132 | parser.add_argument("--download_all", action="store_true", help="Downloads all runs from project", default=False) 133 | parser.add_argument("--wandb_run_id", type=str, help="Specify the wandb run id to download", default=None) 134 | parser.add_argument("--include_tags", type=str, help="Specify the flags to filter the dataset", default=None) 135 | parser.add_argument("--export_mining_dataset", action="store_true", help="Exports the mining dataset", default=False) 136 | parser.add_argument("--export_mining_with_scoring_dataset", action="store_true", help="Exports mining dataset with scores", default=False) 137 | parser.add_argument("--mining_dataset_output_format", type=str, help="Specify the output format of the mining dataset", default="json") 138 | parser.add_argument("--export_path", type=str, help="Specify the path to export the dataset", default="validator_dataset.csv") 139 | parser.add_argument("--blacklist_path", type=str, help="Specify the path to the blacklist phrases", default="blacklist_phrases.txt") 140 | parser.add_argument("--export_openai_dataset", action="store_true", help="Exports the openai dataset", default=False) 141 | 142 | args = parser.parse_args() 143 | 144 | download_all = args.download_all 145 | wandb_run_id = args.wandb_run_id 146 | include_tags = args.include_tags 147 | export_mining_dataset = args.export_mining_dataset 148 | export_mining_with_scoring_dataset = args.export_mining_with_scoring_dataset 149 | export_path = args.export_path 150 | mining_dataset_output_format = args.mining_dataset_output_format 151 | export_openai_dataset = args.export_openai_dataset 152 | 153 | bt.logging.info("Current version of openvalidators: " + openvalidators.__version__) 154 | 155 | # Loads the blacklist phrases into an array 156 | blacklist_phrases = read_file_into_array(args.blacklist_path) 157 | 158 | # Collects dataframe from wandb run logs 159 | collected_data = collect_data(download_all, export_path, wandb_run_id, include_tags) 160 | 161 | # Creates mining dataset 162 | if export_mining_dataset or export_mining_with_scoring_dataset or export_openai_dataset: 163 | create_mining_dataset( 164 | df=collected_data, 165 | export_path=export_path, 166 | mining_dataset_output_format=mining_dataset_output_format, 167 | blacklist_phrases=blacklist_phrases, 168 | with_score=export_mining_with_scoring_dataset, 169 | export_openai_dataset=export_openai_dataset 170 | ) 171 | except Exception as e: 172 | bt.logging.error("Error in training loop", str(e)) 173 | bt.logging.debug(print_exception(value=e)) 174 | -------------------------------------------------------------------------------- /scripts/data_formatter.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import tqdm 3 | import json 4 | from typing import List 5 | from dataclasses import dataclass 6 | 7 | @dataclass 8 | class OpenAISample: 9 | prompt: str 10 | completion: str 11 | 12 | def create_json_dataset( 13 | df: pd.DataFrame, 14 | include_scoring: bool, 15 | blacklist: List[str] 16 | ) -> str: 17 | dict_dataset = {} 18 | 19 | for _, row in tqdm.tqdm(df.iterrows(), desc='Creating mining dataset', total=len(df), unit='run'): 20 | base_prompt = row['base_prompt'] 21 | best_followup = row['best_followup'] 22 | 23 | answer_prompt = row['answer_prompt'] 24 | best_answer = row['best_answer'] 25 | 26 | if best_answer not in blacklist: 27 | if include_scoring: 28 | scores = 0 29 | if isinstance(row["answer_rewards"], list): 30 | scores = max(row["answer_rewards"]) 31 | elif isinstance(row["answer_rewards"], float): 32 | scores = row["answer_rewards"] 33 | 34 | dict_dataset[answer_prompt] = {best_answer: scores} 35 | else: 36 | dict_dataset[answer_prompt] = best_answer 37 | 38 | if best_followup not in blacklist: 39 | if include_scoring: 40 | scores = 0 41 | if isinstance(row["answer_rewards"], list): 42 | scores = max(row["answer_rewards"]) 43 | elif isinstance(row["answer_rewards"], float): 44 | scores = row["answer_rewards"] 45 | dict_dataset[base_prompt] = {best_followup: scores} 46 | else: 47 | dict_dataset[base_prompt] = best_followup 48 | 49 | return dict_dataset 50 | 51 | 52 | def create_csv_dataset( 53 | df: pd.DataFrame, 54 | include_scoring: bool, 55 | blacklist: List[str] 56 | ) -> pd.DataFrame: 57 | if include_scoring: 58 | mining_df = df[['base_prompt', 'best_followup', 'followup_rewards', 'answer_prompt', 'best_answer', 'answer_rewards']] 59 | # Excludes blacklisted phrases from the dataset 60 | filtered_df = mining_df[~df['best_followup'].isin(blacklist)] 61 | filtered_df = filtered_df[~df['best_answer'].isin(blacklist)] 62 | 63 | # Gets the max score for each answer and followup 64 | filtered_df['followup_rewards'] = filtered_df['followup_rewards'].apply(lambda rewards: max(rewards)) 65 | filtered_df['answer_rewards'] = filtered_df['answer_rewards'].apply(lambda rewards: max(rewards)) 66 | 67 | return filtered_df 68 | else: 69 | mining_df = df[['base_prompt', 'best_followup', 'answer_prompt', 'best_answer']] 70 | # Excludes blacklisted phrases from the dataset 71 | filtered_df = mining_df[~df['best_followup'].isin(blacklist)] 72 | filtered_df = filtered_df[~df['best_answer'].isin(blacklist)] 73 | 74 | return filtered_df 75 | 76 | 77 | def create_openai_dataset( 78 | df: pd.DataFrame, 79 | blacklist: List[str] 80 | ) -> str: 81 | samples = [] 82 | 83 | for _, row in tqdm.tqdm(df.iterrows(), desc='Creating openai mining dataset', total=len(df), unit='run'): 84 | base_prompt = row['base_prompt'] 85 | best_followup = row['best_followup'] 86 | 87 | answer_prompt = row['answer_prompt'] 88 | best_answer = row['best_answer'] 89 | 90 | if best_followup not in blacklist: 91 | samples += [OpenAISample(base_prompt, best_followup)] 92 | 93 | if best_answer not in blacklist: 94 | samples += [OpenAISample(answer_prompt, best_answer)] 95 | 96 | # Convert dataclass objects to dictionaries 97 | jsonl_data = "\n".join( 98 | json.dumps({"prompt": sample.prompt, "completion": sample.completion}) 99 | for sample in samples 100 | ) 101 | 102 | return jsonl_data -------------------------------------------------------------------------------- /scripts/release/README.md: -------------------------------------------------------------------------------- 1 | # Release Script(s) Usage 2 | 3 | ## Versioning 4 | 5 | This script needs: 6 | - An existing `openvalidators/__init__.py` file 7 | - An existing `__version__` variable in that file 8 | - An existing version for that variable 9 | 10 | This process will generate: 11 | - A modified version in `__version__` for the update type specified 12 | 13 | ### Example Usage 14 | `./scripts/release/versioning.sh -U patch -A` 15 | 16 | Where: 17 | * `-U` (major|minor|patch) the type of update 18 | * `-A` is to apply the script changes 19 | 20 | 21 | ## Add Notes Changelog 22 | 23 | This script needs: 24 | - An existing `CHANGELOG.md` file with at least three lines 25 | - An existing git tag for the previous version 26 | 27 | This process will generate: 28 | - A new entry in `CHANGELOG.md` 29 | 30 | ##### *Note: This will only list merge commits into the release branch since the last tag* 31 | 32 | ### Example Usage 33 | `./scripts/release/add_notes_changelog.sh -P 1.1.7 -V 1.1.8 -B hotfix/serve-val-axon -T $GIT -A` 34 | 35 | Where: 36 | * `-P` is the old version 37 | * `-V` is the new version 38 | * `-B` is the release branch name (default: `release/vX.X.X`) 39 | * `-T` is the GIT API token 40 | * `-A` is to apply the script changes 41 | 42 | ## Release 43 | 44 | This script needs: 45 | - An existing `__version__` variable in the `openvalidators/__init__.py` file 46 | - Version in the `__version__` variable is not a git tag already 47 | 48 | This process will generate: 49 | - Tag in Github repo: https://github.com/opentensor/validators/tags 50 | - Release in Github: https://github.com/opentensor/validators/releases 51 | 52 | 53 | ### Example Usage 54 | `./scripts/release/release.sh -T $GIT -A` 55 | 56 | Where: 57 | * `-T` is the GIT API token 58 | * `-A` is to apply the script changes 59 | 60 | -------------------------------------------------------------------------------- /scripts/release/add_notes_changelog.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #### 4 | # Utils 5 | #### 6 | source ${BASH_SOURCE%/*}/utils.sh 7 | source ${BASH_SOURCE%/*}/github_utils.sh 8 | ### 9 | 10 | # 1. Get options 11 | 12 | ## Defaults 13 | APPLY="false" 14 | 15 | while [[ $# -gt 0 ]]; do 16 | case $1 in 17 | -A|--apply) 18 | APPLY="true" 19 | shift # past argument 20 | ;; 21 | -P|--previous-version-tag) 22 | PREV_TAG_VERSION="$2" 23 | shift # past argument 24 | shift # past value 25 | ;; 26 | -V|--version) 27 | VERSION="$2" 28 | shift # past argument 29 | shift # past value 30 | ;; 31 | -T|--github-token) 32 | GITHUB_TOKEN="$2" 33 | shift # past argument 34 | shift # past value 35 | ;; 36 | -B|--release-branch) 37 | RELEASE_BRANCH="$2" 38 | shift # past argument 39 | shift # past value 40 | ;; 41 | -*|--*) 42 | echo "Unknown option $1" 43 | exit 1 44 | ;; 45 | *) 46 | POSITIONAL_ARGS+=("$1") # save positional arg 47 | shift # past argument 48 | ;; 49 | esac 50 | done 51 | 52 | if [[ -z $GITHUB_TOKEN && $APPLY == "true" ]]; then 53 | echo_error "Github token required (-T, --github-token)" 54 | exit 1 55 | fi 56 | 57 | if [[ -z $PREV_TAG_VERSION ]]; then 58 | echo_error "Previous version tag required (-P, --previous-version-tag)" 59 | exit 1 60 | fi 61 | 62 | if [[ -z $VERSION ]]; then 63 | echo_error "Version to release required (-V, --version)" 64 | exit 1 65 | fi 66 | 67 | if [[ -z $RELEASE_BRANCH ]]; then 68 | echo_warning "Release branch not specified with (-B, --release-branch) assuming: release/$VERSION" 69 | RELEASE_BRANCH=release/$VERSION 70 | fi 71 | 72 | DATE=$(date +"%Y-%m-%d") 73 | RELEASE_NAME="$VERSION / $DATE" 74 | TAG_NAME=v$VERSION 75 | PREV_TAG_NAME=v$PREV_TAG_VERSION 76 | 77 | # 2.2. Generate release notes 78 | if [[ $APPLY == "true" ]]; then 79 | echo_info "Generating Github release notes" 80 | RESPONSE=$(generate_github_release_notes_for_changelog $GITHUB_TOKEN) 81 | DESCRIPTION=$(echo $RESPONSE | jq '.body' | tail -1 | sed "s/\"//g") 82 | 83 | if [ $(echo $RESPONSE | jq '.body' | wc -l) -eq 1 ]; then 84 | if [ $(echo $RESPONSE | jq '.' | grep 'documentation_url' | wc -l) -gt 0 ]; then 85 | echo_error "Something went wrong generating Github release notes" 86 | echo $RESPONSE | jq --slurp '.[0]' 87 | exit 1 88 | fi 89 | 90 | if [ $(echo $RESPONSE | jq '.type' | grep 'error' | wc -l) -gt 0 ]; then 91 | echo_error "Something went wrong generating Github release notes" 92 | echo $RESPONSE | jq --slurp '.[1]' 93 | exit 1 94 | fi 95 | fi 96 | else 97 | echo_warning "Dry run execution. Not generating Github release notes" 98 | fi 99 | 100 | if [[ $APPLY == "true" ]]; then 101 | echo_info "Adding release notes to CHANGELOG.md" 102 | sed -i "2 i\\\n## $RELEASE_NAME" CHANGELOG.md 103 | sed -i "4 i\\\n$DESCRIPTION\n" CHANGELOG.md 104 | else 105 | echo_warning "Dry run execution. Not adding release notes to CHANGELOG.md" 106 | fi -------------------------------------------------------------------------------- /scripts/release/github_release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #### 4 | # Utils 5 | #### 6 | source ${BASH_SOURCE%/*}/utils.sh 7 | source ${BASH_SOURCE%/*}/github_utils.sh 8 | ### 9 | 10 | # 1. Get options 11 | 12 | ## Defaults 13 | APPLY="false" 14 | 15 | while [[ $# -gt 0 ]]; do 16 | case $1 in 17 | -A|--apply) 18 | APPLY="true" 19 | shift # past argument 20 | ;; 21 | -P|--previous-version-tag) 22 | PREV_TAG_VERSION="$2" 23 | shift # past argument 24 | shift # past value 25 | ;; 26 | -V|--version) 27 | VERSION="$2" 28 | shift # past argument 29 | shift # past value 30 | ;; 31 | -T|--github-token) 32 | GITHUB_TOKEN="$2" 33 | shift # past argument 34 | shift # past value 35 | ;; 36 | -*|--*) 37 | echo "Unknown option $1" 38 | exit 1 39 | ;; 40 | *) 41 | POSITIONAL_ARGS+=("$1") # save positional arg 42 | shift # past argument 43 | ;; 44 | esac 45 | done 46 | 47 | if [[ -z $GITHUB_TOKEN && apply == "true" ]]; then 48 | echo_error "Github token required (-T, --github-token)" 49 | exit 1 50 | fi 51 | 52 | if [[ -z $PREV_TAG_VERSION ]]; then 53 | echo_error "Previous version tag required (-P, --previous-version-tag)" 54 | exit 1 55 | fi 56 | 57 | if [[ -z $VERSION ]]; then 58 | echo_error "Version to release required (-V, --version)" 59 | exit 1 60 | fi 61 | 62 | # 2. Github 63 | DATE=$(date +"%Y-%m-%d") 64 | RELEASE_NAME="$VERSION / $DATE" 65 | PREV_TAG_NAME=$PREV_TAG_VERSION 66 | TAG_NAME=v$VERSION 67 | 68 | # 2.1 Create Git tag for the repository 69 | if [[ $APPLY == "true" ]]; then 70 | echo_info "Tagging repository" 71 | tag_repository $TAG_NAME 72 | else 73 | echo_warning "Dry run execution. Not tagging Github repo" 74 | fi 75 | 76 | # 2.2. Generate release notes 77 | if [[ $APPLY == "true" ]]; then 78 | echo_info "Generating Github release notes" 79 | RESPONSE=$(generate_github_release_notes $GITHUB_TOKEN) 80 | DESCRIPTION=$(echo $RESPONSE | jq '.body' | tail -1 | sed "s/\"//g") 81 | 82 | if [ $(echo $RESPONSE | jq '.body' | wc -l) -eq 1 ]; then 83 | if [ $(echo $RESPONSE | jq '.' | grep 'documentation_url' | wc -l) -gt 0 ]; then 84 | echo_error "Something went wrong generating Github release notes" 85 | echo $RESPONSE | jq --slurp '.[0]' 86 | exit 1 87 | fi 88 | 89 | if [ $(echo $RESPONSE | jq '.type' | grep 'error' | wc -l) -gt 0 ]; then 90 | echo_error "Something went wrong generating Github release notes" 91 | echo $RESPONSE | jq --slurp '.[1]' 92 | exit 1 93 | fi 94 | fi 95 | else 96 | echo_warning "Dry run execution. Not generating Github release notes" 97 | fi 98 | 99 | # 2.3 Create Github release 100 | if [[ $APPLY == "true" ]]; then 101 | echo_info "Generating Github release" 102 | create_github_release $GITHUB_TOKEN 103 | else 104 | echo_warning "Dry run execution. Not creating Github release" 105 | fi -------------------------------------------------------------------------------- /scripts/release/github_utils.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #### 4 | # Utils 5 | #### 6 | source ${BASH_SOURCE%/*}/utils.sh 7 | 8 | # 9 | # Params: 10 | # - First positional argument: version of the tag 11 | # 12 | function tag_repository() 13 | { 14 | VERSION=$1 15 | 16 | if [[ -z $VERSION ]]; then 17 | echo_error "tag_repository needs VERSION" 18 | exit 1 19 | fi 20 | 21 | git tag -a $VERSION -m "Release $VERSION" 22 | git push origin --tags 23 | } 24 | 25 | # 26 | # Params: 27 | # - First positional argument: version of the tag 28 | # 29 | function remove_tag() 30 | { 31 | VERSION=$1 32 | 33 | if [[ -z $VERSION ]]; then 34 | echo_error "remove_tag needs VERSION" 35 | exit 1 36 | fi 37 | 38 | git tag -d $VERSION 39 | git push --delete origin $VERSION 40 | } 41 | 42 | # 43 | # Needs: 44 | # - TAG_NAME 45 | # - PREV_TAG_NAME 46 | # - RELEASE_NAME 47 | # 48 | function generate_github_release_notes_post_data() 49 | { 50 | cat < /dev/null 169 | } -------------------------------------------------------------------------------- /scripts/release/release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # In this script you are going to find the process of releasing Openvalidators. 5 | # 6 | # This script needs: 7 | # - An existing __version__ var in the __init__.py file 8 | # - Version in __version__ var is not a git tag already 9 | # 10 | # This process will generate: 11 | # - Tag in Github repo: https://github.com/opentensor/validators/tags 12 | # - Release in Github: https://github.com/opentensor/validators/releases 13 | # - New entry in CHANGELOG.md file 14 | # 15 | 16 | ### 17 | # Utils 18 | ### 19 | 20 | source ${BASH_SOURCE%/*}/utils.sh 21 | 22 | function help(){ 23 | echo Usage: 24 | echo \ \ $0 25 | echo 26 | echo This script release a openvalidators version. 27 | echo 28 | echo This script needs: 29 | echo \ \ - An existing __version__ var in the __init__.py file 30 | echo \ \ - Version in __version__ var is not a git tag already 31 | echo 32 | } 33 | ### 34 | 35 | ### 36 | # Start of release process 37 | ### 38 | 39 | # 0. Check requirements 40 | # Expected state for the execution environment 41 | # - __version__ exists inside file 'openvalidators/__init__.py' 42 | # - Version has the expected format 43 | 44 | CODE_WITH_VERSION='openvalidators/__init__.py' 45 | 46 | CODE_VERSION=`grep '__version__\ \=\ ' $CODE_WITH_VERSION | awk '{print $3}' | sed 's/"//g'` 47 | VERSION=$CODE_VERSION 48 | 49 | if ! [[ "$CODE_VERSION" =~ ^[0-9]+.[0-9]+.[0-9]+$ ]];then 50 | echo_error "Requirement failure: Version in code '$CODE_VERSION' with wrong format" 51 | exit 1 52 | fi 53 | 54 | # 1. Get options 55 | 56 | ## Defaults 57 | APPLY="false" 58 | APPLY_ACTION="" 59 | 60 | while [[ $# -gt 0 ]]; do 61 | case $1 in 62 | -h|--help) 63 | help 64 | exit 0 65 | ;; 66 | -A|--apply) 67 | APPLY="true" 68 | APPLY_ACTION="--apply" 69 | shift # past argument 70 | ;; 71 | -T|--github-token) 72 | GITHUB_TOKEN="$2" 73 | shift # past argument 74 | shift # past value 75 | ;; 76 | -*|--*) 77 | echo "Unknown option $1" 78 | exit 1 79 | ;; 80 | *) 81 | POSITIONAL_ARGS+=("$1") # save positional arg 82 | shift # past argument 83 | ;; 84 | esac 85 | done 86 | 87 | if [[ $APPLY == "true" ]]; then 88 | echo_warning "Not a Dry run exection" 89 | else 90 | echo_warning "Dry run execution" 91 | fi 92 | 93 | if [[ -z $GITHUB_TOKEN && $APPLY == "true" ]]; then 94 | echo_error "Github token required (-T, --github-token)" 95 | exit 1 96 | fi 97 | 98 | # 2. Checking version 99 | 100 | CURRENT_VERSION_EXISTS=$(git tag | grep $VERSION) 101 | if [[ ! -z $CURRENT_VERSION_EXISTS ]]; then 102 | echo_error "Current version '$VERSION' already exists" 103 | help 104 | exit 1 105 | fi 106 | 107 | PREV_VERSION_TAG=`get_git_tag_higher_version` 108 | 109 | TAG_NAME=v$VERSION 110 | 111 | ## 2.1. Current VERSION is not already a tag 112 | 113 | echo_info "Detected new version tag: $VERSION" 114 | echo_info "Previous version tag: $PREV_VERSION_TAG" 115 | echo_info "Tag generated: $TAG_NAME" 116 | 117 | # 3. Create Github resources 118 | if [[ $APPLY == "true" ]]; then 119 | ${BASH_SOURCE%/*}/github_release.sh $APPLY_ACTION --github-token $GITHUB_TOKEN -P $PREV_VERSION_TAG -V $VERSION 120 | else 121 | ${BASH_SOURCE%/*}/github_release.sh $APPLY_ACTION $GITHUB_TOKEN -P $PREV_VERSION_TAG -V $VERSION 122 | fi 123 | -------------------------------------------------------------------------------- /scripts/release/utils.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | RED='\033[0;31m' 4 | GREEN='\033[0;32m' 5 | YELLOW='\033[0;33m' 6 | NC='\033[0m' # No Color 7 | 8 | function echo_error { 9 | echo -e "${RED}[ERROR]${NC} $1" 10 | } 11 | 12 | function echo_warning { 13 | echo -e "${YELLOW}[WARNING]${NC} $1" 14 | } 15 | 16 | function echo_info { 17 | echo -e "${GREEN}[INFO]${NC} $1" 18 | } 19 | 20 | function echo_json { 21 | echo "{\"type\":\"$1\",\"message\":\"$2\"}" 22 | } 23 | 24 | function get_git_tag_higher_version { 25 | echo `git tag -l --sort -version:refname | head -n 1` 26 | } -------------------------------------------------------------------------------- /scripts/release/versioning.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #### 4 | # Utils 5 | #### 6 | source ${BASH_SOURCE%/*}/utils.sh 7 | ### 8 | 9 | # 1. Get options 10 | 11 | ## Defaults 12 | APPLY="false" 13 | 14 | while [[ $# -gt 0 ]]; do 15 | case $1 in 16 | -A|--apply) 17 | APPLY="true" 18 | shift # past argument 19 | ;; 20 | -U|--update) 21 | VERSION_TYPE="$2" 22 | shift # past argument 23 | shift # past value 24 | ;; 25 | -*|--*) 26 | echo "Unknown option $1" 27 | exit 1 28 | ;; 29 | *) 30 | POSITIONAL_ARGS+=("$1") # save positional arg 31 | shift # past argument 32 | ;; 33 | esac 34 | done 35 | 36 | if [[ $VERSION_TYPE != "major" && $VERSION_TYPE != "minor" && $VERSION_TYPE != "patch" && $VERSION_TYPE != "rc" ]]; then 37 | echo_error "Incorrect version type (-U|--update). Version types accepted: {major, minor, patch}" 38 | exit 1 39 | fi 40 | 41 | VERSION=$(cat VERSION) 42 | CODE_WITH_VERSION='openvalidators/__init__.py' 43 | 44 | MAJOR=$(awk -F. '{print $1}' <<< $VERSION) 45 | MINOR=$(awk -F. '{print $2}' <<< $VERSION) 46 | PATCH=$(awk -F. '{print $3}' <<< $VERSION) 47 | 48 | # RC version 49 | RC=$(awk -F- '{print $NF}' <<< $version) 50 | if [ -z $RC ]; then 51 | CURRENT_VERSION="$MAJOR.$MINOR.$PATCH" 52 | else 53 | CURRENT_VERSION="$MAJOR.$MINOR.$PATCH-$RC" 54 | fi 55 | 56 | case $VERSION_TYPE in 57 | "major") 58 | echo_info "Applying a $VERSION_TYPE update" 59 | NEW_VERSION="$((MAJOR + 1)).0.0" 60 | ;; 61 | "minor") 62 | echo_info "Applying a $VERSION_TYPE update" 63 | NEW_VERSION="$MAJOR.$((MINOR + 1)).0" 64 | ;; 65 | "patch") 66 | echo_info "Applying a $VERSION_TYPE update" 67 | NEW_VERSION="$MAJOR.$MINOR.$((PATCH + 1))" 68 | ;; 69 | "rc") 70 | SUFFIX=$2 71 | if [ -z $SUFFIX ]; then 72 | echo_error "Suffix is needed when updating version to a RC" 73 | exit 1 74 | fi 75 | NEW_VERSION="$MAJOR.$MINOR.$PATCH-$SUFFIX" 76 | ;; 77 | *) 78 | echo_error "This operation is not allowed. Try one of the following: {major, minor, patch, rc}" 79 | exit 1 80 | ;; 81 | esac 82 | 83 | 84 | echo_info "Current version: $CURRENT_VERSION" 85 | echo_info "New version: $NEW_VERSION" 86 | 87 | if [[ $APPLY == "true" ]]; then 88 | echo_info "Updating version in code: sed -i "18,30s/$VERSION/$NEW_VERSION/g" $CODE_WITH_VERSION" 89 | sed -i "18,30s/$VERSION/$NEW_VERSION/g" $CODE_WITH_VERSION 90 | else 91 | echo_warning "Dry run execution. Version update not applied" 92 | echo_info "Use -A or --apply to apply changes" 93 | fi -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | import pathlib 18 | import pkg_resources 19 | from setuptools import setup 20 | 21 | 22 | def read(fname): 23 | this_directory = pathlib.Path(__file__).parent 24 | long_description = (this_directory / fname).read_text() 25 | return long_description 26 | 27 | 28 | def read_requirements(path): 29 | with pathlib.Path(path).open() as requirements_txt: 30 | return [str(requirement) for requirement in pkg_resources.parse_requirements(requirements_txt)] 31 | 32 | 33 | def get_version(rel_path): 34 | for line in read(rel_path).splitlines(): 35 | if line.startswith("__version__"): 36 | delim = '"' if '"' in line else "'" 37 | return line.split(delim)[1] 38 | else: 39 | raise RuntimeError("Unable to find version string.") 40 | 41 | 42 | requirements = read_requirements("requirements.txt") 43 | 44 | 45 | setup( 46 | name="openvalidators", 47 | version=get_version("openvalidators/__init__.py"), 48 | description="Openvalidators is a collection of open source validators for the Bittensor Network.", 49 | url="https://github.com/opentensor/validators", 50 | author="bittensor.com", 51 | packages=["openvalidators"], 52 | include_package_data=True, 53 | author_email="", 54 | license="MIT", 55 | long_description=read("README.md"), 56 | long_description_content_type="text/markdown", 57 | entry_points={ 58 | "console_scripts": ["foundation-validator = openvalidators.neuron:main"], 59 | }, 60 | install_requires=requirements, 61 | python_requires=">=3.8", 62 | classifiers=[ 63 | "Intended Audience :: Developers", 64 | "Topic :: Software Development :: Build Tools", 65 | "License :: OSI Approved :: MIT License", 66 | "Programming Language :: Python :: 3 :: Only", 67 | "Programming Language :: Python :: 3.8", 68 | "Programming Language :: Python :: 3.9", 69 | "Programming Language :: Python :: 3.10", 70 | "Topic :: Scientific/Engineering", 71 | "Topic :: Scientific/Engineering :: Mathematics", 72 | "Topic :: Scientific/Engineering :: Artificial Intelligence", 73 | "Topic :: Software Development", 74 | "Topic :: Software Development :: Libraries", 75 | "Topic :: Software Development :: Libraries :: Python Modules", 76 | ], 77 | maintainer="", 78 | maintainer_email="", 79 | keywords=[ 80 | "bittensor", 81 | "validator", 82 | "ai", 83 | "machine-learning", 84 | "deep-learning", 85 | "blockchain", 86 | "pytorch", 87 | "torch", 88 | "neural-networks", 89 | "cryptocurrency", 90 | ], 91 | ) 92 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | -------------------------------------------------------------------------------- /tests/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2023 Opentensor Technologies 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | 18 | from bittensor_wallet.mock import MockWallet as _MockWallet, utils as _mock_wallet_utils 19 | 20 | _get_mock_coldkey = _mock_wallet_utils.get_mock_coldkey 21 | _get_mock_hotkey = _mock_wallet_utils.get_mock_hotkey 22 | _get_mock_keypair = _mock_wallet_utils.get_mock_keypair 23 | _get_mock_wallet = _mock_wallet_utils.get_mock_wallet 24 | 25 | 26 | def __mock_wallet_factory__(*args, **kwargs) -> _MockWallet: 27 | """Returns a mock wallet object.""" 28 | 29 | mock_wallet = _get_mock_wallet() 30 | 31 | return mock_wallet 32 | -------------------------------------------------------------------------------- /tests/reward/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opentensor/validators/9e2172f3889faa9ae5bc7dbae08ba4291610a022/tests/reward/__init__.py -------------------------------------------------------------------------------- /tests/reward/test_task_validator.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from openvalidators.reward.task_validator import TaskValidator 3 | 4 | class TaskValidatorTestCase(unittest.TestCase): 5 | """ 6 | This class contains unit tests for the TaskValidator class. 7 | 8 | The tests cover different scenarios for the `reward` method of the TaskValidator class. 9 | The `reward` method is expected to return a reward based on the task name and the completion text. 10 | """ 11 | 12 | def setUp(self): 13 | self.validator = TaskValidator() 14 | 15 | def test_augment_with_answer_keyword(self): 16 | """ 17 | Test if the reward method returns 0 when the task "name" starts with 'augment' (summarization) 18 | and the completion contains the 'Answer:' keyword. 19 | """ 20 | name = f'augment' 21 | completion = "Summary: test summary\nAnswer: Test answer" 22 | self.assertEqual(self.validator.reward('', completion, name), 0.0) 23 | 24 | def test_followup_with_answer_keyword(self): 25 | """ 26 | Test if the reward method returns 0 when the task "name" starts with 'followup' (question generation) 27 | and the completion contains the 'Answer:' keyword. 28 | """ 29 | for i in range(0, 4): 30 | name = f'followup{i}' 31 | completion = 'Question: This is a test question?\nAnswer: This is a test answer.' 32 | self.assertEqual(self.validator.reward('', completion, name), 0.0) 33 | 34 | def test_augment_with_question_keyword(self): 35 | """ 36 | Test if the reward method returns 0 when the task "name" starts with 'augment' (summarization) 37 | and the completion contains the 'Question:' keyword. 38 | """ 39 | name = f'augment' 40 | completion = "Summary: test summary\nQuestion: This is a test question?" 41 | self.assertEqual(self.validator.reward('', completion, name), 0.0) 42 | 43 | def test_answer_with_question_keyword(self): 44 | """ 45 | Test if the reward method returns 0 when the task "name" is 'answer' (answer generation) 46 | and the completion contains the 'Question:' keyword. 47 | """ 48 | for i in range(0, 4): 49 | name = f'answer{i}' 50 | completion = 'Question: This is a test question?\nAnswer: This is a test answer.' 51 | self.assertEqual(self.validator.reward('', completion, name), 0.0) 52 | 53 | def test_followup_and_answer_with_summary_keyword(self): 54 | """ 55 | Test if the reward method returns 0 when the task "name" is different from "augment" (summarization) 56 | and the completion contains the 'Summary:' keyword. 57 | """ 58 | for name in ['followup0', 'followup1', 'followup2', 'followup3', 'answer0', 'answer1', 'answer2', 'answer3']: 59 | completion = 'Summary: This is a test summary.' 60 | self.assertEqual(self.validator.reward('', completion, name), 0.0) 61 | 62 | def test_reward_valid_followup(self): 63 | """ 64 | Test if the reward method returns 1 when the task "name" starts with 'followup' (question generation) 65 | and the completion contains a question 66 | """ 67 | for i in range(0, 4): 68 | name = f'followup{i}' 69 | completion = 'Question: This is a test question?' 70 | self.assertEqual(self.validator.reward('', completion, name), 1.0) 71 | 72 | def test_reward_valid_answer(self): 73 | """ 74 | Test if the reward method returns 1 when the task "name" is 'answer' (answer generation) 75 | and the completion contains an answer 76 | """ 77 | for i in range(0, 4): 78 | name = f'answer{i}' 79 | completion = 'Answer: This is a test answer.' 80 | self.assertEqual(self.validator.reward('', completion, name), 1.0) 81 | 82 | def test_reward_valid_augment(self): 83 | """ 84 | Test if the reward method returns 1 when the task "name" is 'augment' (summarization) 85 | and the completion contains the a summary. 86 | """ 87 | name = 'augment' 88 | completion = 'Summary: This is a test summary.' 89 | self.assertEqual(self.validator.reward('', completion, name), 1.0) 90 | 91 | def test_reward_valid_other(self): 92 | """ 93 | Test if the reward method returns 1 when the task "name" is different from "augment", "followup", and "answer" 94 | and the completion does not contain the 'Summary:', 'Answer:', and 'Question:' keywords. 95 | """ 96 | for name in ['followup0', 'followup1', 'followup2', 'followup3', 'answer0', 'answer1', 'answer2', 'answer3']: 97 | completion = 'This is a test completion.' 98 | self.assertEqual(self.validator.reward('', completion, name), 1.0) 99 | 100 | if __name__ == '__main__': 101 | unittest.main() -------------------------------------------------------------------------------- /tests/test_dataset.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from openvalidators.dataset import Dataset 3 | 4 | 5 | class DatasetTestCase(unittest.TestCase): 6 | def test_next_skips_empty_and_newline_only_strings(self): 7 | mock_data = iter([{"text": ""}, {"text": "\n\n"}, {"text": "Non-empty text"}]) 8 | dataset = Dataset() 9 | dataset.openwebtext = mock_data 10 | dataset.red_pajama = mock_data 11 | 12 | # Test that __next__ skips empty texts and texts that consist only of newline characters 13 | self.assertEqual(dataset.__next__(), {"text": "Non-empty text"}) 14 | 15 | def test_next_returns_regular_strings(self): 16 | mock_data = iter([{"text": "Non-empty text"}]) 17 | dataset = Dataset() 18 | dataset.openwebtext = mock_data 19 | dataset.red_pajama = mock_data 20 | 21 | # Test that __next__ returns a non-empty text 22 | self.assertEqual(dataset.__next__(), {"text": "Non-empty text"}) 23 | 24 | 25 | if __name__ == '__main__': 26 | unittest.main() -------------------------------------------------------------------------------- /tests/test_dendrite.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | import torch 18 | import bittensor as bt 19 | import copy 20 | import unittest 21 | from unittest.mock import MagicMock 22 | from openvalidators.dendrite import AsyncDendritePool 23 | 24 | 25 | class DendriteTestCase(unittest.TestCase): 26 | def setUp(self): 27 | """ 28 | Creates a mock metagraph with 1024 mock axons. 29 | """ 30 | mock_metagraph = MagicMock(spec=bt.metagraph) 31 | mock_metagraph.uids = torch.tensor(range(0, 1024)) 32 | mock_metagraph.axons = [ 33 | bt.axon_info( 34 | version=0, 35 | ip="0.0.0.0/0", 36 | port=12345, 37 | ip_type=0, 38 | hotkey=str(num), 39 | coldkey=str(num) 40 | ) for num in range(0, 1024) 41 | ] 42 | 43 | self.metagraph = mock_metagraph 44 | self.keypair = "test" 45 | 46 | def test_resync_uid_modified_metagraph(self): 47 | # Arrange: Creates Async dendrite pool and modifies the metagraph by changing the axon_info at defined index 48 | dendrite_pool = AsyncDendritePool(keypair=self.keypair, metagraph=self.metagraph) 49 | 50 | # Modify the hotkey of the first axon of the metagraph 51 | index = 0 52 | modified_metagraph = copy.deepcopy(self.metagraph) 53 | modified_metagraph.axons[index].hotkey = "hotkey-test" 54 | 55 | # Act: Resync the dendrite pool with the modified metagraph 56 | dendrite_pool.resync(modified_metagraph) 57 | 58 | # Assert: The dendrite pool hotkeys should be the same as the modified metagraph hotkeys after resync 59 | dendrite_hot_keys = list(map(lambda dendrite: dendrite.axon_info.hotkey, dendrite_pool.dendrites)) 60 | new_metagraph_hot_keys = list(map(lambda axon: axon.hotkey, modified_metagraph.axons)) 61 | 62 | self.assertEqual(dendrite_hot_keys, new_metagraph_hot_keys) 63 | 64 | def test_resync_uid_add(self): 65 | original_metagraph = self.metagraph 66 | 67 | smaller_metagraph = copy.deepcopy(original_metagraph) 68 | 69 | # Remove the last axon from the metagraph 70 | smaller_metagraph.axons.pop() 71 | 72 | # Creates dendrite pool with smaller metagraph 73 | dendrite_pool = AsyncDendritePool(keypair=self.keypair, metagraph=smaller_metagraph) 74 | 75 | # Resync the dendrite pool with the original metagraph, that has one more axon 76 | dendrite_pool.resync(original_metagraph) 77 | 78 | assert len(dendrite_pool.dendrites) == len(original_metagraph.axons) 79 | 80 | dendrite_hot_keys = list(map(lambda dendrite: dendrite.axon_info.hotkey, dendrite_pool.dendrites)) 81 | new_metagraph_hot_keys = list(map(lambda axon: axon.hotkey, original_metagraph.axons)) 82 | 83 | self.assertEqual(dendrite_hot_keys, new_metagraph_hot_keys) 84 | 85 | 86 | if __name__ == "__main__": 87 | unittest.main() 88 | -------------------------------------------------------------------------------- /tests/test_event.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | import unittest 18 | from dataclasses import fields 19 | from unittest.mock import patch 20 | from openvalidators.event import EventSchema 21 | from openvalidators.reward import RewardModelType 22 | 23 | class EventTestCase(unittest.TestCase): 24 | 25 | def test_event_from_dict_all_forward_columns_match(self): 26 | """Test that all default columns logged on the forward pass are correctly converted 27 | """ 28 | # Arrange: Create a dictionary with all columns 29 | event_dict = { 30 | 'completions': ['test'], 31 | 'completion_times': [0.123], 32 | 'name': 'test-name', 33 | 'block': 1.0, 34 | 'gating_loss': 1.0, 35 | 'uids': [1], 36 | 'prompt': 'test-prompt', 37 | 'step_length': 1.0, 38 | 'best': 'test-best', 39 | 'rewards': [1.0], 40 | RewardModelType.dahoas.value: [1.0], 41 | RewardModelType.blacklist.value: [1.0], 42 | RewardModelType.nsfw.value: [1.0], 43 | RewardModelType.reciprocate.value: [1.0], 44 | RewardModelType.diversity.value: [1.0], 45 | RewardModelType.dpo.value: [1.0], 46 | RewardModelType.rlhf.value: [1.0], 47 | RewardModelType.prompt.value: [1.0], 48 | RewardModelType.relevance.value: [1.0], 49 | RewardModelType.task_validator.value: [1.0], 50 | 51 | RewardModelType.dahoas.value + '_normalized': [1.0], 52 | RewardModelType.blacklist.value + '_normalized': [1.0], 53 | RewardModelType.nsfw.value + '_normalized': [1.0], 54 | RewardModelType.reciprocate.value + '_normalized': [1.0], 55 | RewardModelType.diversity.value + '_normalized': [1.0], 56 | RewardModelType.dpo.value + '_normalized': [1.0], 57 | RewardModelType.rlhf.value + '_normalized': [1.0], 58 | RewardModelType.prompt.value + '_normalized': [1.0], 59 | RewardModelType.relevance.value + '_normalized': [1.0], 60 | RewardModelType.task_validator.value + '_normalized': [1.0] 61 | } 62 | 63 | # Act 64 | with patch('bittensor.logging.warning') as mock_warning: 65 | event = EventSchema.from_dict(event_dict, disable_log_rewards=False) 66 | mock_warning.assert_not_called() 67 | 68 | # Assert 69 | for field in fields(EventSchema): 70 | field_name = field.name 71 | field_value = getattr(event, field_name) 72 | 73 | # Note: Does not include 'set_weights' column as it is not logged on the forward pass 74 | if field_name == 'set_weights': 75 | assert field_value is None 76 | continue 77 | 78 | print(field_name, field_value) 79 | assert field_name in event_dict and event_dict[field_name] == field_value 80 | 81 | 82 | def test_event_from_dict_forward_no_reward_logging(self): 83 | """Test that all default columns (not including reward columns) logged on the forward pass are 84 | correctly converted""" 85 | # Assert: create a dictionary with all non-related reward columns 86 | event_dict = { 87 | 'completions': ['test'], 88 | 'completion_times': [0.123], 89 | 'name': 'test-name', 90 | 'block': 1.0, 91 | 'gating_loss': 1.0, 92 | 'uids': [1], 93 | 'prompt': 'test-prompt', 94 | 'step_length': 1.0, 95 | 'best': 'test-best', 96 | 'rewards': [1.0], 97 | } 98 | 99 | # Act 100 | with patch('bittensor.logging.warning') as mock_warning: 101 | event = EventSchema.from_dict(event_dict, disable_log_rewards=True) 102 | mock_warning.assert_not_called() 103 | 104 | # Assert: Check that all columns that were logged are correctly converted 105 | for key, value in event_dict.items(): 106 | assert getattr(event, key) == value 107 | 108 | # Assert: Check that all reward columns that are not logged are set to None 109 | assert event.set_weights is None 110 | assert event.dahoas_reward_model is None 111 | assert event.blacklist_filter is None 112 | assert event.nsfw_filter is None 113 | assert event.reciprocate_reward_model is None 114 | assert event.diversity_reward_model is None 115 | assert event.dpo_reward_model is None 116 | assert event.rlhf_reward_model is None 117 | assert event.prompt_reward_model is None 118 | assert event.relevance_filter is None 119 | assert event.task_validator_filter is None 120 | 121 | assert event.dahoas_reward_model_normalized is None 122 | assert event.nsfw_filter_normalized is None 123 | assert event.reciprocate_reward_model_normalized is None 124 | assert event.diversity_reward_model_normalized is None 125 | assert event.dpo_reward_model_normalized is None 126 | assert event.rlhf_reward_model_normalized is None 127 | assert event.prompt_reward_model_normalized is None 128 | assert event.relevance_filter_normalized is None 129 | assert event.task_validator_filter_normalized is None 130 | 131 | def test_event_from_dict_forward_reward_logging_mismatch(self): 132 | """Test that all default columns logged on the forward pass are correctly converted and that 133 | that reward columns that should be logged are logged as warnings""" 134 | # Assert: create a dictionary with all non-related reward columns 135 | event_dict = { 136 | 'completions': ['test'], 137 | 'completion_times': [0.123], 138 | 'name': 'test-name', 139 | 'block': 1.0, 140 | 'gating_loss': 1.0, 141 | 'uids': [1], 142 | 'prompt': 'test-prompt', 143 | 'step_length': 1.0, 144 | 'best': 'test-best', 145 | 'rewards': [1.0], 146 | } 147 | 148 | not_logged_columns = [] 149 | for field in RewardModelType: 150 | not_logged_columns.append(field.value) 151 | if field.value != 'blacklist_filter': 152 | not_logged_columns.append(field.value + '_normalized') 153 | 154 | 155 | # Act 156 | with patch('bittensor.logging.warning') as mock_warning: 157 | event = EventSchema.from_dict(event_dict, disable_log_rewards=False) 158 | # Assert: Check that all columns that are not logged in the dict are logged as warnings 159 | self.assertEqual(mock_warning.call_count, len(not_logged_columns)) 160 | 161 | # Assert: Check that all columns that were logged are correctly converted 162 | for key, value in event_dict.items(): 163 | assert getattr(event, key) == value 164 | 165 | # Assert: Check that all reward columns that are not logged are set to None 166 | assert event.set_weights is None 167 | assert event.dahoas_reward_model is None 168 | assert event.blacklist_filter is None 169 | assert event.nsfw_filter is None 170 | assert event.reciprocate_reward_model is None 171 | assert event.diversity_reward_model is None 172 | assert event.dpo_reward_model is None 173 | assert event.rlhf_reward_model is None 174 | assert event.prompt_reward_model is None 175 | assert event.relevance_filter is None 176 | assert event.task_validator_filter is None 177 | 178 | assert event.dahoas_reward_model_normalized is None 179 | assert event.nsfw_filter_normalized is None 180 | assert event.reciprocate_reward_model_normalized is None 181 | assert event.diversity_reward_model_normalized is None 182 | assert event.dpo_reward_model_normalized is None 183 | assert event.rlhf_reward_model_normalized is None 184 | assert event.prompt_reward_model_normalized is None 185 | assert event.relevance_filter_normalized is None 186 | assert event.task_validator_filter_normalized is None 187 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | import torch 18 | import bittensor as bt 19 | import copy 20 | import unittest 21 | from unittest.mock import MagicMock 22 | from openvalidators.utils import resync_linear_layer, check_uid_availability 23 | 24 | 25 | class UtilsTestCase(unittest.TestCase): 26 | def setUp(self): 27 | """ 28 | Creates a mock metagraph with 1024 mock axons before each test. 29 | """ 30 | mock_metagraph = MagicMock(spec=bt.metagraph) 31 | mock_metagraph.uids = torch.tensor(range(0, 1024)) 32 | mock_metagraph.S = torch.zeros(1024) 33 | mock_metagraph.hotkeys = list(map(str, range(0, 1024))) 34 | mock_metagraph.validator_permit = [False] * 1024 35 | mock_metagraph.axons = [ 36 | MagicMock(spec=bt.axon_info, hotkey=str(num), ip="0.0.0.0/0", port=12345) for num in range(0, 1024) 37 | ] 38 | 39 | self.metagraph = mock_metagraph 40 | self.keypair = "test" 41 | 42 | def test_resync_linear_layer_multiple_updates(self): 43 | # Arrange: Create necessary inputs for the test 44 | # Create a linear layer of 768 x uids full of ones 45 | linear_output_size = len(self.metagraph.uids) 46 | linear_layer = torch.nn.Linear(768, linear_output_size) 47 | torch.nn.init.ones_(linear_layer.weight) 48 | torch.nn.init.ones_(linear_layer.bias) 49 | 50 | # Create a new metagraph state with updated hotkeys 51 | updated_uids_indices = [0, 10, 20, 30] 52 | modified_metagraph = copy.deepcopy(self.metagraph) 53 | 54 | for modified_index in updated_uids_indices: 55 | modified_metagraph.hotkeys[modified_index] = "test" 56 | 57 | # Act: Call the utils function to be tested 58 | resync_linear_layer(linear_layer, self.metagraph, modified_metagraph) 59 | 60 | # Assert: Ensure that the bias of the updated indices have been reinitialized as expected 61 | for index in range(0, linear_output_size): 62 | # If the index has been updated, assert that bias is zero and weights are not ones 63 | if index in updated_uids_indices: 64 | self.assertEqual(linear_layer.bias[index].item(), 0) 65 | self.assertFalse(torch.all(linear_layer.weight[index] == torch.ones(linear_layer.weight[index].shape))) 66 | # If the index has not been updated, assert that bias is one and weights are ones 67 | else: 68 | self.assertEqual(linear_layer.bias[index].item(), 1) 69 | self.assertTrue(torch.all(linear_layer.weight[index] == torch.ones(linear_layer.weight[index].shape))) 70 | 71 | def test_check_uid_availability_not_serving_axon(self): 72 | # Arrange: Create a non serving axon 73 | uid = 1 74 | self.metagraph.axons[uid] = MagicMock(spec=bt.axon_info, is_serving=False) 75 | 76 | # Act: Call the function to check if uid is available 77 | result = check_uid_availability(self.metagraph, uid, vpermit_tao_limit=0) 78 | 79 | # Assert: Ensure that the result is False (uid is available) when node doesn't have a serving axon 80 | self.assertFalse(result) 81 | 82 | def test_check_uid_availability_node_without_validator_permit(self): 83 | # Arrange: Create a serving axon without validator permit 84 | uid = 1 85 | self.metagraph.axons[uid] = MagicMock(spec=bt.axon_info, is_serving=True) 86 | self.metagraph.validator_permit[uid] = False 87 | 88 | # Act: Call the function to check if uid is available 89 | result = check_uid_availability(self.metagraph, uid, vpermit_tao_limit=0) 90 | 91 | # Assert: Ensure that the result is True (uid is available) when node does not have a validator permit 92 | self.assertTrue(result) 93 | 94 | def test_check_uid_availability_validator_with_stake_less_than_vpermit_tao_limit(self): 95 | # Arrange: Create a serving axon with validator permit and stake less than vpermit_tao_limit 96 | uid = 1 97 | self.metagraph.axons[uid] = MagicMock(spec=bt.axon_info, is_serving=True) 98 | self.metagraph.validator_permit[uid] = True 99 | self.metagraph.S[uid] = 1 100 | v_permit_tao_limit = 2 101 | 102 | # Act: Call the function to check if uid is available 103 | result = check_uid_availability(self.metagraph, uid, vpermit_tao_limit=v_permit_tao_limit) 104 | 105 | # Assert: Ensure that the result is True (uid is available) when node validator 106 | # has stake less than vpermit_tao_limit 107 | self.assertTrue(result) 108 | 109 | def test_check_uid_availability_validator_with_stake_greater_than_vpermit_tao_limit(self): 110 | # Arrange: Create a serving axon with validator permit and stake greater than vpermit_tao_limit 111 | uid = 1 112 | self.metagraph.axons[uid] = MagicMock(spec=bt.axon_info, is_serving=True) 113 | self.metagraph.validator_permit[uid] = True 114 | self.metagraph.S[uid] = 2 115 | v_permit_tao_limit = 1 116 | 117 | # Act: Call the function to check if uid is available 118 | result = check_uid_availability(self.metagraph, uid, vpermit_tao_limit=v_permit_tao_limit) 119 | 120 | # Assert: Ensure that the result is False (uid is available) when validator node 121 | # has stake greater than vpermit_tao_limit 122 | self.assertFalse(result) 123 | 124 | 125 | if __name__ == "__main__": 126 | unittest.main() 127 | -------------------------------------------------------------------------------- /tests/test_weights.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2021 Yuma Rao 3 | 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | # documentation files (the “Software”), to deal in the Software without restriction, including without limitation 6 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 7 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | 9 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 10 | # the Software. 11 | 12 | # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 13 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | # DEALINGS IN THE SOFTWARE. 17 | import torch 18 | import copy 19 | import asyncio 20 | import sys 21 | from openvalidators.neuron import neuron as Neuron 22 | from openvalidators.forward import run_step 23 | from unittest.mock import MagicMock, patch 24 | 25 | from .helpers import __mock_wallet_factory__ 26 | 27 | CLI_ARGS_STR = "validators/openvalidators/neuron.py --mock --wallet._mock --wandb.off --neuron.followup_sample_size 10 --neuron.answer_sample_size 10" 28 | 29 | SYS_ARGV = sys.argv.copy() 30 | 31 | 32 | patcher = None 33 | 34 | def setUpModule(): 35 | """Runs once for the tests in this module.""" 36 | global patcher 37 | patcher = patch("bittensor.wallet.__new__", __mock_wallet_factory__ ) 38 | patcher.start() 39 | 40 | def tearDownModule(): 41 | """Runs once for the tests in this module.""" 42 | global patcher 43 | if patcher: 44 | patcher.stop() 45 | 46 | 47 | def test_uid_weights_unchanged_unless_queried(n_steps=10, n_concurrent=1): 48 | """Test that the weights of unqueried uids do not over the course of a forward pass.""" 49 | 50 | sys.argv = CLI_ARGS_STR.split(" ") 51 | neuron = Neuron() 52 | neuron.blacklist = MagicMock(return_value =True) 53 | 54 | for _ in range(n_steps): 55 | 56 | prev_scores = copy.deepcopy(neuron.moving_averaged_scores) 57 | 58 | async def forward(neuron): 59 | # Request a run step 60 | event = await run_step( 61 | neuron, 62 | prompt = '', 63 | name = 'augment', 64 | k = 10, 65 | timeout = 10, 66 | ) 67 | return event 68 | 69 | # run concurrent forward passes 70 | async def run_forward(): 71 | coroutines = [forward(neuron) for _ in range(n_concurrent)] 72 | return await asyncio.gather(*coroutines) 73 | 74 | events = neuron.loop.run_until_complete(run_forward()) 75 | # moving_averaged_scores updates are not thread safe, so I don't think we can run concurrent forwards 76 | for event in events: 77 | 78 | # get current scores 79 | next_scores = copy.deepcopy(neuron.moving_averaged_scores) 80 | 81 | queried_uids = sorted(set(event["uids"])) 82 | ignored_uids = [uid for uid in torch.arange(neuron.metagraph.n.item()) if uid not in queried_uids] 83 | 84 | # ther is a floating point difference (~1e-10) between the scores, so we can't use exact equality 85 | assert next_scores[ignored_uids].allclose(prev_scores[ignored_uids]), "Unqueried uids should not change" 86 | 87 | sys.argv = SYS_ARGV.copy() 88 | --------------------------------------------------------------------------------