├── VERSION ├── neurons ├── __init__.py ├── discriminator │ └── __init__.py ├── generator │ ├── __init__.py │ ├── services │ │ ├── __init__.py │ │ ├── base_service.py │ │ ├── service_registry.py │ │ └── stabilityai_service.py │ └── task_manager.py ├── validator │ ├── __init__.py │ └── services │ │ └── __init__.py └── base.py ├── gas ├── generation │ ├── tps │ │ ├── __init__.py │ │ └── nano_banana.py │ ├── util │ │ ├── __init__.py │ │ ├── prompt.py │ │ └── image.py │ ├── __init__.py │ └── model_registry.py ├── scraping │ ├── __init__.py │ └── base.py ├── datasets │ ├── __init__.py │ └── datasets.py ├── __init__.py ├── evaluation │ ├── __init__.py │ └── miner_type_tracker.py ├── cache │ ├── __init__.py │ ├── util │ │ └── __init__.py │ └── types.py ├── protocol │ ├── __init__.py │ ├── webhooks.py │ ├── encoding.py │ ├── epistula.py │ └── model_uploads.py ├── verification │ ├── __init__.py │ └── c2pa_verification.py ├── utils │ ├── __init__.py │ ├── chain_model_metadata_store.py │ ├── model_zips.py │ ├── utils.py │ ├── wandb_utils.py │ ├── metagraph.py │ └── autoupdater.py └── types.py ├── docs ├── static │ ├── bm-logo.png │ ├── bm-logo-black.png │ ├── threshold_decay.png │ ├── Join-BitMind-Discord.png │ └── GAS-Architecture-Simple.png ├── Mining.md ├── Installation.md ├── Validating.md ├── Discriminative-Mining.md ├── ONNX.md ├── Generative-Mining.md └── Incentive.md ├── .env.validator.template ├── .gitignore ├── LICENSE ├── pyproject.toml ├── min_compute.yml ├── .env.gen_miner.template ├── README.md ├── tests └── generator │ └── stabilityai_service.py ├── gen_miner.config.js └── validator.config.js /VERSION: -------------------------------------------------------------------------------- 1 | 4.3.3 2 | -------------------------------------------------------------------------------- /neurons/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gas/generation/tps/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gas/generation/util/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /neurons/discriminator/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /neurons/generator/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /neurons/validator/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /neurons/generator/services/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gas/scraping/__init__.py: -------------------------------------------------------------------------------- 1 | from .google import GoogleScraper -------------------------------------------------------------------------------- /docs/static/bm-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BitMind-AI/bitmind-subnet/HEAD/docs/static/bm-logo.png -------------------------------------------------------------------------------- /docs/static/bm-logo-black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BitMind-AI/bitmind-subnet/HEAD/docs/static/bm-logo-black.png -------------------------------------------------------------------------------- /docs/static/threshold_decay.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BitMind-AI/bitmind-subnet/HEAD/docs/static/threshold_decay.png -------------------------------------------------------------------------------- /docs/static/Join-BitMind-Discord.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BitMind-AI/bitmind-subnet/HEAD/docs/static/Join-BitMind-Discord.png -------------------------------------------------------------------------------- /gas/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | from .datasets import ( 2 | load_all_datasets, 3 | get_image_datasets, 4 | get_video_datasets, 5 | ) -------------------------------------------------------------------------------- /docs/static/GAS-Architecture-Simple.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BitMind-AI/bitmind-subnet/HEAD/docs/static/GAS-Architecture-Simple.png -------------------------------------------------------------------------------- /gas/generation/__init__.py: -------------------------------------------------------------------------------- 1 | from .generation_pipeline import GenerationPipeline 2 | from .prompt_generator import PromptGenerator 3 | from .models import initialize_model_registry 4 | -------------------------------------------------------------------------------- /gas/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "4.3.3" 2 | 3 | version_split = __version__.split(".") 4 | __spec_version__ = ( 5 | (100000 * int(version_split[0])) 6 | + (1000 * int(version_split[1])) 7 | + (10 * int(version_split[2])) 8 | ) 9 | -------------------------------------------------------------------------------- /gas/evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | from .generative_challenge_manager import GenerativeChallengeManager 2 | from .miner_type_tracker import MinerTypeTracker 3 | from .rewards import ( 4 | get_discriminator_rewards, 5 | get_generator_base_rewards, 6 | get_generator_reward_multipliers 7 | ) -------------------------------------------------------------------------------- /gas/cache/__init__.py: -------------------------------------------------------------------------------- 1 | from .content_manager import ContentManager 2 | from .media_storage import MediaStorage 3 | from .types import Media 4 | from .content_db import ContentDB 5 | from .types import PromptEntry, MediaEntry 6 | 7 | __all__ = ["ContentManager", "MediaStorage", "ContentDB", "PromptEntry", "MediaEntry", "Media"] 8 | -------------------------------------------------------------------------------- /neurons/validator/services/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Services module for SN34 validator. 3 | 4 | This module contains standalone services that can run independently 5 | from the main validator process for better resource isolation and log management. 6 | """ 7 | 8 | from .data_service import DataService 9 | from .generator_service import GeneratorService 10 | 11 | __all__ = ["DataService", "GeneratorService"] -------------------------------------------------------------------------------- /gas/protocol/__init__.py: -------------------------------------------------------------------------------- 1 | from .epistula import ( 2 | generate_header, 3 | verify_signature, 4 | create_header_hook, 5 | get_verifier, 6 | determine_epistula_version_and_verify, 7 | ) 8 | 9 | from .validator_requests import ( 10 | get_miner_type, 11 | query_generative_miner, 12 | ) 13 | 14 | from .encoding import ( 15 | image_to_bytes, 16 | video_to_bytes, 17 | media_to_bytes, 18 | ) -------------------------------------------------------------------------------- /docs/Mining.md: -------------------------------------------------------------------------------- 1 | # Mining Guide 2 | 3 | GAS supports two types of miners that work together in an adversarial loop: 4 | 5 | ## [Discriminative Mining](Discriminative-Mining.md) 📖 6 | Miners run classifiers that detect AI-generated content. They receive images or videos and predict whether the media is real, fully synthetic, or partially modified by AI. 7 | 8 | ## [Generative Mining](Generative-Mining.md) 🎨 9 | Miners create synthetic media that challenges the discriminators. They generate increasingly realistic content to test and improve detection capabilities. 10 | 11 | --- 12 | 13 | **Choose your path above to get started with mining on GAS.** -------------------------------------------------------------------------------- /.env.validator.template: -------------------------------------------------------------------------------- 1 | # ======= Validator Configuration (FILL IN) ======= 2 | # Wallet 3 | WALLET_NAME= 4 | WALLET_HOTKEY= 5 | 6 | # API Keys 7 | WANDB_API_KEY= 8 | HUGGINGFACE_HUB_TOKEN= 9 | OPEN_ROUTER_API_KEY= 10 | 11 | # Network 12 | CHAIN_ENDPOINT=wss://entrypoint-finney.opentensor.ai:443 13 | # OTF public finney endpoint: wss://entrypoint-finney.opentensor.ai:443 14 | # OTF public testnet endpoint: wss://test.finney.opentensor.ai:443/ 15 | 16 | CALLBACK_PORT= 17 | #EXTERNAL_CALLBACK_PORT= # Optional 18 | 19 | # Cache config 20 | SN34_CACHE_DIR=~/.cache/sn34 21 | HF_HOME=~/.cache/huggingface 22 | HEARTBEAT=true 23 | 24 | # Generator config 25 | GENERATION_BATCH_SIZE=3 26 | DEVICE=cuda 27 | 28 | # Other 29 | LOGLEVEL=info 30 | AUTO_UPDATE=true 31 | -------------------------------------------------------------------------------- /gas/cache/util/__init__.py: -------------------------------------------------------------------------------- 1 | from gas.cache.util.filesystem import ( 2 | is_source_complete, 3 | is_zip_complete, 4 | is_parquet_complete, 5 | get_most_recent_update_time, 6 | extract_media_info, 7 | format_to_extension, 8 | get_format_from_content, 9 | ) 10 | 11 | 12 | 13 | from gas.cache.util.video import ( 14 | get_video_duration, 15 | get_video_metadata, 16 | seconds_to_str, 17 | ) 18 | 19 | __all__ = [ 20 | # Filesystem 21 | "is_source_complete", 22 | "is_zip_complete", 23 | "is_parquet_complete", 24 | "get_most_recent_update_time", 25 | "extract_media_info", 26 | "format_to_extension", 27 | "get_format_from_content", 28 | 29 | # Video 30 | "get_video_duration", 31 | "get_video_metadata", 32 | "seconds_to_str", 33 | ] 34 | -------------------------------------------------------------------------------- /gas/verification/__init__.py: -------------------------------------------------------------------------------- 1 | from .verification_pipeline import run_verification, verify_media, get_verification_summary 2 | from .clip_utils import ( 3 | preload_clip_models, 4 | clear_clip_models, 5 | calculate_clip_alignment, 6 | calculate_clip_alignment_consensus, 7 | ) 8 | from .duplicate_detection import ( 9 | compute_image_hash, 10 | compute_video_hash, 11 | compute_media_hash, 12 | compute_crop_resistant_hash, 13 | hamming_distance, 14 | count_crop_segment_matches, 15 | is_duplicate, 16 | find_duplicates, 17 | check_duplicate_in_db, 18 | DEFAULT_HAMMING_THRESHOLD, 19 | DEFAULT_CROP_RESISTANT_MATCH_THRESHOLD, 20 | ) 21 | from .c2pa_verification import ( 22 | verify_c2pa, 23 | is_from_trusted_generator, 24 | C2PAVerificationResult, 25 | TRUSTED_ISSUERS, 26 | C2PA_AVAILABLE, 27 | ) -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Python 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | *.so 6 | .Python 7 | build/ 8 | develop-eggs/ 9 | dist/ 10 | downloads/ 11 | eggs/ 12 | .eggs/ 13 | lib/ 14 | lib64/ 15 | parts/ 16 | sdist/ 17 | var/ 18 | wheels/ 19 | *.egg-info/ 20 | .installed.cfg 21 | *.egg 22 | .coverage 23 | *.ipynb 24 | uv.lock 25 | 26 | # Virtual environments 27 | venv/ 28 | env/ 29 | ENV/ 30 | .venv/ 31 | 32 | # IDE 33 | .vscode/ 34 | .idea/ 35 | *.swp 36 | *.swo 37 | 38 | # OS 39 | .DS_Store 40 | Thumbs.db 41 | 42 | # Logs 43 | *.log 44 | 45 | # Environment variables 46 | .env 47 | .env.local 48 | .env.development 49 | .env.test 50 | .env.production 51 | 52 | # Model files and cache 53 | *.pth 54 | *.pt 55 | *.ckpt 56 | .cache/ 57 | 58 | # Bittensor specific 59 | ~/.bittensor/ 60 | *.wallet 61 | 62 | .env.validator 63 | .env.miner 64 | .env.generator 65 | .env.mock_miner 66 | 67 | *.onnx 68 | downloaded_models/* 69 | 70 | .env.gen_miner 71 | base_miner/test_output/* 72 | base_miner/.env.gen_miner 73 | *.onnx 74 | downloaded_models/* 75 | 76 | generated_content/* 77 | 78 | wandb/* 79 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright © 2023 Yuma Rao 3 | Copyright © 2025 BitMind 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 6 | documentation files (the "Software"), to deal in the Software without restriction, including without limitation 7 | the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 8 | and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of 11 | the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 14 | THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 15 | THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 16 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 17 | DEALINGS IN THE SOFTWARE. 18 | -------------------------------------------------------------------------------- /gas/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .utils import ( 2 | print_info, 3 | fail_with_none, 4 | on_block_interval, 5 | ExitContext, 6 | get_metadata, 7 | get_file_modality, 8 | run_in_thread, 9 | ) 10 | 11 | from .metagraph import ( 12 | get_miner_uids, 13 | create_set_weights, 14 | ) 15 | 16 | from .autoupdater import autoupdate 17 | 18 | from .transforms import ( 19 | apply_random_augmentations, 20 | get_base_transforms, 21 | get_random_augmentations, 22 | get_random_augmentations_medium, 23 | get_random_augmentations_hard, 24 | ) 25 | 26 | from .state_manager import ( 27 | StateManager, 28 | save_validator_state, 29 | load_validator_state, 30 | ) 31 | 32 | __all__ = [ 33 | # Core utilities 34 | "print_info", 35 | "fail_with_none", 36 | "on_block_interval", 37 | "ExitContext", 38 | "get_metadata", 39 | "get_file_modality", 40 | "run_in_thread", 41 | # Metagraph utilities 42 | "get_miner_uids", 43 | "create_set_weights", 44 | # Autoupdater 45 | "autoupdate", 46 | # Transforms 47 | "apply_random_augmentations", 48 | "get_base_transforms", 49 | "get_random_augmentations", 50 | "get_random_augmentations_medium", 51 | "get_random_augmentations_hard", 52 | # State management 53 | "StateManager", 54 | "save_validator_state", 55 | "load_validator_state", 56 | ] -------------------------------------------------------------------------------- /gas/generation/util/prompt.py: -------------------------------------------------------------------------------- 1 | def get_tokenizer_with_min_len(model): 2 | """ 3 | Returns the tokenizer with the smallest maximum token length. 4 | 5 | Args: 6 | model: Single pipeline or dict of pipeline stages. 7 | 8 | Returns: 9 | tuple: (tokenizer, max_token_length) 10 | """ 11 | # Get the model to check for tokenizers 12 | pipeline = model["stage1"] if isinstance(model, dict) else model 13 | 14 | # If model has two tokenizers, return the one with smaller max length 15 | if hasattr(pipeline, "tokenizer_2"): 16 | len_1 = pipeline.tokenizer.model_max_length 17 | len_2 = pipeline.tokenizer_2.model_max_length 18 | return ( 19 | (pipeline.tokenizer_2, len_2) 20 | if len_2 < len_1 21 | else (pipeline.tokenizer, len_1) 22 | ) 23 | 24 | return pipeline.tokenizer, pipeline.tokenizer.model_max_length 25 | 26 | 27 | def truncate_prompt_if_too_long(prompt: str, model): 28 | """ 29 | Truncates the input string if it exceeds the maximum token length when tokenized. 30 | 31 | Args: 32 | prompt (str): The text prompt that may need to be truncated. 33 | 34 | Returns: 35 | str: The original prompt if within the token limit; otherwise, a truncated version of the prompt. 36 | """ 37 | tokenizer, max_token_len = get_tokenizer_with_min_len(model) 38 | tokens = tokenizer(prompt, verbose=False) # Suppress token max exceeded warnings 39 | if len(tokens["input_ids"]) < max_token_len: 40 | return prompt 41 | 42 | # Truncate tokens if they exceed the maximum token length, decode the tokens back to a string 43 | truncated_prompt = tokenizer.decode( 44 | token_ids=tokens["input_ids"][: max_token_len - 1], skip_special_tokens=True 45 | ) 46 | tokens = tokenizer(truncated_prompt) 47 | return truncated_prompt 48 | -------------------------------------------------------------------------------- /gas/datasets/datasets.py: -------------------------------------------------------------------------------- 1 | """ 2 | Dataset definitions from gasbench for the validator cache system 3 | """ 4 | 5 | from typing import List 6 | from gas.types import DatasetConfig, MediaType, Modality 7 | from gasbench.dataset.config import load_benchmark_datasets_from_yaml 8 | 9 | 10 | 11 | def _from_benchmark_config(bc) -> DatasetConfig: 12 | """ 13 | Map gasbench BenchmarkDatasetConfig -> bitmind-subnet DatasetConfig 14 | """ 15 | return DatasetConfig( 16 | path=bc.path, 17 | modality=bc.modality, # coerced to Modality in DatasetConfig.__post_init__ 18 | media_type=bc.media_type, # coerced to MediaType in DatasetConfig.__post_init__ 19 | source_format=(bc.source_format or ""), 20 | ) 21 | 22 | 23 | def _load_yaml_datasets(modality: str) -> List[DatasetConfig]: 24 | """ 25 | Load datasets from gasbench's bundled benchmark_datasets.yaml and map them. 26 | Filters out unsupported sources for current downloader (e.g., gasstation/*, non-HF sources). 27 | """ 28 | data = load_benchmark_datasets_from_yaml() 29 | bench_list = data.get(modality, []) or [] 30 | 31 | mapped: List[DatasetConfig] = [] 32 | for bc in bench_list: 33 | # Exclude gasstation paths and non-HuggingFace sources (bitmind-subnet downloader only supports HF) 34 | try: 35 | source = getattr(bc, "source", "huggingface") 36 | path = getattr(bc, "path", "") or "" 37 | if source != "huggingface": 38 | continue 39 | if path.startswith("gasstation/"): 40 | continue 41 | mapped.append(_from_benchmark_config(bc)) 42 | except Exception: 43 | continue 44 | 45 | return mapped 46 | 47 | 48 | def get_image_datasets() -> List[DatasetConfig]: 49 | return _load_yaml_datasets("image") 50 | 51 | 52 | def get_video_datasets() -> List[DatasetConfig]: 53 | return _load_yaml_datasets("video") 54 | 55 | 56 | def load_all_datasets() -> List[DatasetConfig]: 57 | return get_image_datasets() + get_video_datasets() 58 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=69.0", "wheel", "pip>=21.0"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "gas" 7 | dynamic = ["version"] 8 | description = "SN34 on bittensor" 9 | authors = [{ name = "GAS", email = "intern@bitmind.ai" }] 10 | readme = "README.md" 11 | requires-python = ">=3.10" 12 | license = { text = "" } 13 | urls = { homepage = "http://bitmind.ai" } 14 | 15 | dependencies = [ 16 | "bittensor==9.9.0", 17 | "bittensor-cli", 18 | "pillow==10.4.0", 19 | "substrate-interface==1.7.11", 20 | "numpy==2.0.1", 21 | "pandas==2.3.0", 22 | "torch==2.7.1", 23 | "torchvision==0.22.1", 24 | #"flash-attn>=2.7.0", 25 | "asyncpg==0.30.0", 26 | "httpcore==1.0.7", 27 | "httpx==0.28.1", 28 | "pyarrow==19.0.1", 29 | "ffmpeg-python==0.2.0", 30 | "bitsandbytes==0.45.4", 31 | "black==25.1.0", 32 | "pre-commit==4.2.0", 33 | # "diffusers==0.33.1", 34 | "transformers==4.55.0", 35 | "qwen-vl-utils>=0.0.14", 36 | "scikit-learn==1.7.1", 37 | "av==14.2.0", 38 | "opencv-python==4.11.0.86", 39 | "wandb==0.19.9", 40 | "uvicorn==0.27.1", 41 | "python-multipart==0.0.20", 42 | "peft==0.17.0", 43 | "aiohttp>=3.10.2", 44 | "onnx==1.18.0", 45 | "sympy==1.14.0", 46 | "coloredlogs==15.0.1", 47 | "flatbuffers==25.2.10", 48 | "humanfriendly==10.0", 49 | "hf_xet==1.1.7", 50 | "nest-asyncio==1.6.0", 51 | "python-dotenv==1.0.1", 52 | "accelerate==1.8.1", 53 | "selenium==4.34.0", 54 | "stamina==25.1.0", 55 | "click>=8.0.0", 56 | "datasets==4.0.0", 57 | "async-substrate-interface>=1.5.1", 58 | "ftfy==6.3.1", 59 | "huggingface_hub>=0.17.0", 60 | "gasbench[gpu] @ git+https://github.com/BitMind-AI/gasbench.git@main", 61 | "imagehash>=4.3.1", 62 | "c2pa-python>=0.25.0", 63 | ] 64 | 65 | [project.scripts] 66 | gascli = "gas.cli:cli" 67 | 68 | [tool.setuptools] 69 | packages = { find = { where = [ 70 | ".", 71 | ], exclude = [ 72 | "docs*", 73 | "wandb*", 74 | "*.egg-info", 75 | ] } } 76 | 77 | [tool.setuptools.dynamic] 78 | version = { file = "VERSION" } 79 | 80 | -------------------------------------------------------------------------------- /gas/utils/chain_model_metadata_store.py: -------------------------------------------------------------------------------- 1 | import functools 2 | from typing import Optional 3 | 4 | import bittensor as bt 5 | 6 | from gas.types import DiscriminatorModelId as ModelId, DiscriminatorModelMetadata as ModelMetadata 7 | from gas.utils import run_in_thread 8 | 9 | 10 | class ChainModelMetadataStore: 11 | """Chain based implementation for storing and retrieving metadata about a model.""" 12 | 13 | def __init__( 14 | self, 15 | subtensor: bt.subtensor, 16 | netuid: int, 17 | ): 18 | self.subtensor = subtensor 19 | self.netuid = netuid 20 | 21 | async def store_model_metadata( 22 | self, 23 | wallet: str, 24 | model_id: ModelId, 25 | wait_for_inclusion: bool = True, 26 | wait_for_finalization: bool = True, 27 | ttl: int = 60, 28 | ): 29 | """Stores model metadata on this subnet for a specific wallet.""" 30 | data = model_id.to_compressed_str() 31 | 32 | commit_partial = functools.partial( 33 | self.subtensor.commit, 34 | wallet, 35 | self.netuid, 36 | data, 37 | ) 38 | 39 | run_in_thread(commit_partial, ttl) 40 | 41 | async def retrieve_model_metadata( 42 | self, uid: int, hotkey: str, ttl: int = 60 43 | ) -> Optional[ModelMetadata]: 44 | """Retrieves model metadata on this subnet for specific hotkey""" 45 | 46 | metadata_partial = functools.partial( 47 | bt.core.extrinsics.serving.get_metadata, 48 | self.subtensor, 49 | self.netuid, 50 | hotkey, 51 | ) 52 | 53 | commitment_partial = functools.partial( 54 | self.subtensor.get_commitment, 55 | self.netuid, 56 | uid, 57 | ) 58 | 59 | metadata = run_in_thread(metadata_partial, ttl) 60 | 61 | if not metadata: 62 | return None 63 | 64 | chain_str = run_in_thread(commitment_partial, ttl) 65 | 66 | model_id = None 67 | bt.logging.info(f"chain_str: {chain_str}") 68 | try: 69 | model_id = ModelId.from_compressed_str(chain_str) 70 | except: 71 | bt.logging.trace( 72 | f"Failed to parse the metadata on the chain for hotkey {hotkey}." 73 | ) 74 | return None 75 | 76 | model_metadata = ModelMetadata(id=model_id, block=metadata["block"]) 77 | 78 | return model_metadata 79 | -------------------------------------------------------------------------------- /min_compute.yml: -------------------------------------------------------------------------------- 1 | # NOTE FOR MINERS: 2 | # Miner min compute varies based on selected model architecture. 3 | # For model training, you will most likely need a GPU. For miner deployment, depending 4 | # on your model, you may be able to get away with CPU. 5 | 6 | version: '3.0.0' 7 | 8 | compute_spec: 9 | 10 | validator: 11 | 12 | cpu: 13 | min_cores: 4 # Minimum number of CPU cores 14 | min_speed: 2.5 # Minimum speed per core (GHz) 15 | recommended_cores: 8 # Recommended number of CPU cores 16 | recommended_speed: 3.5 # Recommended speed per core (GHz) 17 | architecture: "x86_64" # Architecture type (e.g., x86_64, arm64) 18 | 19 | gpu: 20 | required: True # Does the application require a GPU? 21 | min_vram: 80 # Minimum GPU VRAM (GB) 22 | recommended_vram: 80 # Recommended GPU VRAM (GB) 23 | min_compute_capability: 8.0 # Minimum CUDA compute capability 24 | recommended_compute_capability: 8.0 # Recommended CUDA compute capability 25 | recommended_gpu: "NVIDIA A100 80GB PCIE" # Recommended GPU to purchase/rent 26 | fp64: 9.7 # TFLOPS 27 | fp64_tensor_core: 19.5 # TFLOPS 28 | fp32: 19.5 # TFLOPS 29 | tf32: 156 # TFLOPS* 30 | bfloat16_tensor_core: 312 # TFLOPS* 31 | int8_tensor_core: 624 # TOPS* 32 | 33 | # See NVIDIA A100 datasheet for details: 34 | # https://www.nvidia.com/content/dam/en-zz/Solutions/Data-Center/a100/pdf/ 35 | # nvidia-a100-datasheet-nvidia-us-2188504-web.pdf 36 | 37 | # *double with sparsity 38 | 39 | memory: 40 | min_ram: 32 # Minimum RAM (GB) 41 | min_swap: 4 # Minimum swap space (GB) 42 | recommended_swap: 8 # Recommended swap space (GB) 43 | ram_type: "DDR6" # RAM type (e.g., DDR4, DDR3, etc.) 44 | 45 | storage: 46 | min_space: 1000 # Minimum free storage space (GB) 47 | recommended_space: 1000 # Recommended free storage space (GB) 48 | type: "SSD" # Preferred storage type (e.g., SSD, HDD) 49 | min_iops: 1000 # Minimum I/O operations per second (if applicable) 50 | recommended_iops: 5000 # Recommended I/O operations per second 51 | 52 | os: 53 | name: "Ubuntu" # Name of the preferred operating system(s) 54 | version: 22.04 # Version of the preferred operating system(s) 55 | 56 | network_spec: 57 | bandwidth: 58 | download: 100 # Minimum download bandwidth (Mbps) 59 | upload: 20 # Minimum upload bandwidth (Mbps) 60 | -------------------------------------------------------------------------------- /neurons/generator/services/base_service.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from typing import Dict, Any, Optional 3 | import bittensor as bt 4 | 5 | from ..task_manager import GenerationTask 6 | 7 | 8 | class BaseGenerationService(ABC): 9 | """ 10 | Abstract base class for generation services. 11 | 12 | This defines the interface that all generation services must implement, 13 | whether they use 3rd party APIs or local models. 14 | """ 15 | 16 | def __init__(self, config: Any = None): 17 | self.config = config 18 | self.name = self.__class__.__name__ 19 | bt.logging.info(f"Initializing {self.name}") 20 | 21 | @abstractmethod 22 | def is_available(self) -> bool: 23 | """Check if this service is available (e.g., API keys set, models loaded).""" 24 | pass 25 | 26 | @abstractmethod 27 | def supports_modality(self, modality: str) -> bool: 28 | """Check if this service supports the given modality (e.g., image, video).""" 29 | pass 30 | 31 | @abstractmethod 32 | def process(self, task: GenerationTask) -> Dict[str, Any]: 33 | """ 34 | Process a generation task and return the result. 35 | 36 | Args: 37 | task: The generation task to process 38 | 39 | Returns: 40 | Dict containing: 41 | - 'data': Optional[bytes] - Binary result data (if generated locally) 42 | - 'url': Optional[str] - Direct URL to result (if from 3rd party) 43 | - 'metadata': Optional[Dict] - Any additional metadata 44 | 45 | Raises: 46 | Exception: If processing fails 47 | """ 48 | pass 49 | 50 | def get_info(self) -> Dict[str, Any]: 51 | """Get information about this service.""" 52 | return { 53 | "name": self.name, 54 | "available": self.is_available(), 55 | "supported_tasks": self.get_supported_tasks() 56 | } 57 | 58 | @abstractmethod 59 | def get_supported_tasks(self) -> Dict[str, list]: 60 | """Return dict of supported task types by modality.""" 61 | pass 62 | 63 | @abstractmethod 64 | def get_api_key_requirements(self) -> Dict[str, str]: 65 | """ 66 | Return dict of required environment variables and their descriptions. 67 | 68 | Returns: 69 | Dict with env var names as keys and human-readable descriptions as values. 70 | Example: { 71 | 'OPENAI_API_KEY': 'OpenAI API key for DALL-E image generation', 72 | 'OPENAI_ORG_ID': 'OpenAI organization ID (optional)' 73 | } 74 | """ 75 | pass 76 | -------------------------------------------------------------------------------- /gas/utils/model_zips.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import os 3 | import zipfile 4 | from pathlib import Path 5 | 6 | 7 | def calculate_sha256(data: bytes) -> str: 8 | return hashlib.sha256(data).hexdigest() 9 | 10 | 11 | def validate_onnx_directory(onnx_dir: str) -> bool: 12 | """Validate that the directory contains all required ONNX files""" 13 | 14 | required_files = [ 15 | "image_detector.onnx", 16 | "video_detector.onnx", 17 | ] 18 | 19 | if not os.path.exists(onnx_dir): 20 | print(f"Error: Directory does not exist: {onnx_dir}") 21 | return False 22 | 23 | if not os.path.isdir(onnx_dir): 24 | print(f"Error: Path is not a directory: {onnx_dir}") 25 | return False 26 | 27 | missing_files = [] 28 | for filename in required_files: 29 | file_path = os.path.join(onnx_dir, filename) 30 | if not os.path.exists(file_path): 31 | missing_files.append(filename) 32 | 33 | if missing_files: 34 | print(f"Error: Missing required ONNX files: {missing_files}") 35 | print(f"Expected files in {onnx_dir}:") 36 | for filename in required_files: 37 | print(f" - {filename}") 38 | return False 39 | 40 | print(f"✅ Found all required ONNX files in {onnx_dir}:") 41 | for filename in required_files: 42 | file_path = os.path.join(onnx_dir, filename) 43 | file_size = os.path.getsize(file_path) 44 | print(f" - {filename} ({file_size} bytes)") 45 | 46 | return True 47 | 48 | 49 | def create_model_zip(onnx_dir): 50 | """Create zip file containing multiple ONNX models 51 | 52 | Args: 53 | onnx_dir: Path to directory containing ONNX model files 54 | 55 | Raises: 56 | FileNotFoundError: If ONNX directory or files do not exist 57 | ValueError: If required files missing 58 | """ 59 | 60 | if not os.path.exists(onnx_dir): 61 | raise FileNotFoundError(f"ONNX directory not found: {onnx_dir}") 62 | 63 | required_files = [ 64 | "image_detector.onnx", 65 | "video_detector.onnx", 66 | ] 67 | missing_files = [] 68 | for filename in required_files: 69 | file_path = os.path.join(onnx_dir, filename) 70 | if not os.path.exists(file_path): 71 | missing_files.append(filename) 72 | 73 | if missing_files: 74 | raise FileNotFoundError(f"Missing required ONNX files: {missing_files}") 75 | 76 | zip_path = os.path.join(onnx_dir, "models.zip") 77 | with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zipf: 78 | for filename in required_files: 79 | file_path = os.path.join(onnx_dir, filename) 80 | zipf.write(file_path, filename) 81 | 82 | return zip_path 83 | 84 | 85 | -------------------------------------------------------------------------------- /docs/Installation.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | ## Prerequisites 4 | 5 | 1. **Python 3.10+** (required) 6 | 2. **uv** (fast Python package manager) 7 | 3. **Git** (for cloning the repository) 8 | 9 | ### System Dependencies 10 | 11 | The installation script will automatically install the following system dependencies (unless `--no-system-deps` is specified): 12 | 13 | - **Build tools**: pkg-config, cmake 14 | - **Media processing**: ffmpeg 15 | - **Browser automation**: Google Chrome, libnss3, libnspr4, xvfb 16 | - **Process management**: Node.js, npm, PM2, dotenv 17 | 18 | **Note**: The `--no-system-deps` option is primarily intended for **discriminative miners** who only need to submit models and don't require Chrome browser automation or Node.js process management tools. Validators should use the full installation to ensure all dependencies are available. 19 | 20 | ### Installing uv 21 | 22 | Install uv using one of these methods: 23 | 24 | ```bash 25 | # Method 1: Official installer (recommended) 26 | curl -LsSf https://astral.sh/uv/install.sh | sh 27 | 28 | # Method 2: Using pip 29 | pip install uv 30 | ``` 31 | 32 | ## Installation 33 | 34 | 1. **Clone the repository**: 35 | ```bash 36 | git clone 37 | cd bitmind-subnet 38 | ``` 39 | 40 | 2. **Run the installation script**: 41 | ```bash 42 | ./install.sh 43 | ``` 44 | 45 | **Options:** 46 | - `./install.sh --no-system-deps` - Skip system dependency installation (intended for discriminative miners) 47 | 48 | The installation script will: 49 | - Check for Python 3.10+ and uv 50 | - Install system dependencies (unless `--no-system-deps` is specified): 51 | - pkg-config, cmake, ffmpeg 52 | - chrome web driver, libnss3, libnspr4, xvfb 53 | - Node.js, npm, PM2, dotenv 54 | - Create a virtual environment using uv (fast dependency resolution) 55 | - Install all dependencies from `pyproject.toml` 56 | - Install the GAS package in development mode 57 | - Install the `gascli` command-line tool 58 | - Install additional git dependencies (Janus) 59 | 60 | ## Usage 61 | 62 | ### Activating the Virtual Environment 63 | 64 | Before using `gascli`, you need to activate the virtual environment: 65 | 66 | ```bash 67 | source .venv/bin/activate 68 | ``` 69 | 70 | ### CLI Commands 71 | 72 | Once the virtual environment is activated, you can use the GAS CLI: 73 | 74 | ```bash 75 | gascli --help # Show main help 76 | gascli validator --help # Validator commands help 77 | gascli miner --help # Miner commands help 78 | ``` 79 | 80 | ### Available Aliases 81 | 82 | - `validator` → `vali`, `v` 83 | - `miner` → `m` 84 | 85 | ### Global Commands 86 | 87 | ```bash 88 | # Show all services status 89 | gascli status 90 | ``` 91 | 92 | ### Alternative Usage (without activation) 93 | 94 | You can also run commands directly without activating the environment: 95 | 96 | ```bash 97 | .venv/bin/gascli --help 98 | .venv/bin/gascli validator start 99 | ``` 100 | -------------------------------------------------------------------------------- /docs/Validating.md: -------------------------------------------------------------------------------- 1 | # Validator Guide 2 | 3 | ## Before You Proceed 4 | 5 | Follow the [Installation Guide](Installation.md) to set up your environment before proceeding with validator setup. 6 | 7 | 8 | ## Setup Instructions 9 | 10 | > Once you've run the installation script, create a `.env.validator` file in the project root. 11 | 12 | ```bash 13 | $ cp .env.validator.template .env.validator 14 | ``` 15 | 16 | ```bash 17 | # ======= Validator Configuration (FILL IN) ======= 18 | # Wallet 19 | WALLET_NAME= 20 | WALLET_HOTKEY= 21 | 22 | # API Keys 23 | WANDB_API_KEY= 24 | HUGGINGFACE_HUB_TOKEN= 25 | 26 | # Network 27 | CHAIN_ENDPOINT=wss://entrypoint-finney.opentensor.ai:443 28 | # OTF public finney endpoint: wss://entrypoint-finney.opentensor.ai:443 29 | # OTF public testnet endpoint: wss://test.finney.opentensor.ai:443/ 30 | 31 | # Benchmark API (optional - defaults to https://gas.bitmind.ai) 32 | BENCHMARK_API_URL=https://gas.bitmind.ai 33 | 34 | # Cache config 35 | SN34_CACHE_DIR=~/.cache/sn34 36 | HEARTBEAT=true 37 | 38 | # Generator config 39 | GENERATION_BATCH_SIZE=3 40 | DEVICE=cuda 41 | 42 | # Other 43 | LOGLEVEL=info 44 | AUTO_UPDATE=true 45 | ``` 46 | 47 | > Once you've populated `.env.validator`, activate the virtual environment and start your validator processes 48 | ```bash 49 | $ source .venv/bin/activate 50 | $ gascli validator start 51 | ``` 52 | The above command will create 3 pm2 processes: 53 | ```bash 54 | ┌────┬───────────────────┬─────────────┬─────────┬─────────┬──────────┬────────┬──────┬───────────┬──────────┬──────────┬──────────┬──────────┐ 55 | │ id │ name │ namespace │ version │ mode │ pid │ uptime │ ↺ │ status │ cpu │ mem │ user │ watching │ 56 | ├────┼───────────────────┼─────────────┼─────────┼─────────┼──────────┼────────┼──────┼───────────┼──────────┼──────────┼──────────┼──────────┤ 57 | │ 2 │ sn34-data │ default │ N/A │ fork │ 4032914 │ 2s │ 72 │ online │ 100% │ 529.0mb │ user │ disabled │ 58 | │ 1 │ sn34-generator │ default │ N/A │ fork │ 4032936 │ 2s │ 72 │ online │ 100% │ 448.5mb │ user │ disabled │ 59 | │ 0 │ sn34-validator │ default │ N/A │ fork │ 4032918 │ 2s │ 72 │ online │ 100% │ 504.0mb │ user │ disabled │ 60 | └────┴───────────────────┴─────────────┴─────────┴─────────┴──────────┴────────┴──────┴───────────┴──────────┴──────────┴──────────┴──────────┘ 61 | ``` 62 | - **sn34-data**: Handles data downloads 63 | - **sn34-generator**: Responsible for generating prompts and synthetic/semisynthetic media 64 | - **sn34-validator**: Core validator logic. Challenges, scoring, weight setting. 65 | 66 | 67 | ## Validator Operations 68 | 69 | First, activate the virtual environment: 70 | ```bash 71 | source .venv/bin/activate 72 | ``` 73 | 74 | Then run validator commands: 75 | ```bash 76 | # Start validator services 77 | gascli validator start 78 | gascli v start # Using alias 79 | gascli v stop 80 | gascli v status 81 | gascli v logs 82 | gascli v --help 83 | ``` 84 | 85 | -------------------------------------------------------------------------------- /.env.gen_miner.template: -------------------------------------------------------------------------------- 1 | # ============================================================================= 2 | # GenerativeMiner Environment Configuration 3 | # ============================================================================= 4 | # Copy this file to .env.miner and configure your settings 5 | # cp .env.gen_miner.template .env.gen_miner 6 | 7 | # ============================================================================= 8 | # GENERATION SERVICE SELECTION 9 | # ============================================================================= 10 | # Choose which service to use for each modality: 11 | # openai - DALL-E 3 (requires OPENAI_API_KEY) 12 | # openrouter - Google Gemini via OpenRouter (requires OPEN_ROUTER_API_KEY) 13 | # local - Local Stable Diffusion models (requires GPU) 14 | # 15 | # If not set, loads ALL available services (wastes memory if using API services) 16 | IMAGE_SERVICE= 17 | VIDEO_SERVICE= 18 | 19 | 20 | # ============================================================================= 21 | # API KEYS (configure the services you want to use) 22 | # ============================================================================= 23 | 24 | # OpenAI DALL-E 25 | OPENAI_API_KEY= 26 | 27 | # OpenRouter (multi-model access) 28 | OPEN_ROUTER_API_KEY= 29 | 30 | # Hugging Face (for model downloads) 31 | HUGGINGFACE_HUB_TOKEN= 32 | 33 | # Google Cloud (if using Vertex AI) 34 | # GOOGLE_CLOUD_PROJECT= 35 | # GOOGLE_CLOUD_LOCATION=us-central1 36 | # GOOGLE_APPLICATION_CREDENTIALS= 37 | 38 | # ============================================================================= 39 | # BITTENSOR NETWORK 40 | # ============================================================================= 41 | 42 | # Network Configuration 43 | BT_NETUID=379 44 | BT_CHAIN_ENDPOINT=wss://test.finney.opentensor.ai:443 45 | 46 | # Wallet 47 | BT_WALLET_NAME=miner1 48 | BT_WALLET_HOTKEY=default 49 | 50 | # Axon (your miner's endpoint) 51 | BT_AXON_PORT=8093 52 | BT_AXON_IP=0.0.0.0 53 | # BT_AXON_EXTERNAL_IP=your.public.ip 54 | 55 | # ============================================================================= 56 | # MINER SETTINGS 57 | # ============================================================================= 58 | 59 | # Performance 60 | MINER_DEVICE=auto 61 | MINER_MAX_CONCURRENT_TASKS=5 62 | MINER_TASK_TIMEOUT=300 63 | 64 | # Storage 65 | HF_HOME=/workspace/.cache/huggingface 66 | 67 | # Set to true to save generated images/videos to MINER_OUTPUT_DIR 68 | MINER_SAVE_LOCALLY=true 69 | MINER_OUTPUT_DIR=./miner_generated_content 70 | 71 | 72 | # ============================================================================= 73 | # LOGGING 74 | # ============================================================================= 75 | 76 | BT_LOGGING_LEVEL=INFO 77 | 78 | # ============================================================================= 79 | # USAGE EXAMPLES 80 | # ============================================================================= 81 | 82 | # Start miner: gascli generator start 83 | # Check status: gascli generator status 84 | # View logs: gascli generator logs -f 85 | # Direct PM2: pm2 start miner.config.js 86 | 87 | # Get API keys: 88 | # - OpenAI: https://platform.openai.com/api-keys 89 | # - OpenRouter: https://openrouter.ai/keys 90 | # - Hugging Face: https://huggingface.co/settings/tokens -------------------------------------------------------------------------------- /gas/generation/tps/nano_banana.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import base64 3 | import os 4 | import time 5 | import io 6 | from PIL import Image 7 | from gas.types import Modality, MediaType 8 | 9 | 10 | def generate_image(prompt: str, model: str = "google/gemini-2.5-flash-image-preview:free"): 11 | """ 12 | Generates an image based on the given prompt using an API call. 13 | Raises an exception if the API key is not present. 14 | 15 | Example response structure: 16 | 17 | "choices": [ 18 | { 19 | "message": { 20 | "role": "assistant", 21 | "content": "I've generated a beautiful sunset image for you.", 22 | "images": [ 23 | { 24 | "type": "image_url", 25 | "image_url": { 26 | "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA..." 27 | } 28 | } 29 | ] 30 | } 31 | } 32 | ] 33 | 34 | 35 | Args: 36 | prompt (str): The text prompt to generate the image. 37 | model (str): The model to use for image generation. Defaults to a specific model. 38 | 39 | Returns: 40 | Dict shaped like GenerationPipeline output or None on failure, e.g.: 41 | { 42 | "image": PIL.Image.Image, 43 | "modality": Modality.IMAGE, 44 | "media_type": MediaType.SYNTHETIC, 45 | "prompt": str, 46 | "model_name": str, 47 | "time": float, 48 | "gen_duration": float, 49 | "gen_args": dict 50 | } 51 | """ 52 | API_KEY = os.getenv("OPEN_ROUTER_API_KEY") 53 | if not API_KEY: 54 | raise RuntimeError("API key is not set. Please set the 'OPEN_ROUTER_API_KEY' environment variable in .env.validator.") 55 | 56 | URL = "https://openrouter.ai/api/v1/chat/completions" 57 | 58 | headers = { 59 | "Authorization": f"Bearer {API_KEY}", 60 | "Content-Type": "application/json", 61 | } 62 | 63 | payload = { 64 | "model": model, 65 | "messages": [ 66 | { 67 | "role": "user", 68 | "content": prompt 69 | } 70 | ], 71 | "modalities": ["image", "text"] 72 | } 73 | 74 | start_time = time.time() 75 | response = requests.post(URL, headers=headers, json=payload) 76 | 77 | if response.status_code != 200: 78 | return None 79 | 80 | result = response.json() 81 | 82 | if 'choices' not in result or not result['choices']: 83 | return None 84 | 85 | choice = result['choices'][0] 86 | if 'message' not in choice or 'images' not in choice['message']: 87 | return None 88 | 89 | images = choice['message']['images'] 90 | if not images: 91 | return None 92 | 93 | # Use the first image 94 | image_url = images[0]['image_url']['url'] 95 | # Format: "data:image/;base64," 96 | if ',' not in image_url: 97 | return None 98 | base64_data = image_url.split(',')[1] 99 | image_binary = base64.b64decode(base64_data) 100 | pil_image = Image.open(io.BytesIO(image_binary)) 101 | 102 | gen_time = time.time() - start_time 103 | 104 | output = { 105 | "image": pil_image, 106 | "modality": Modality.IMAGE, 107 | "media_type": MediaType.SYNTHETIC, 108 | "prompt": prompt, 109 | "model_name": model, 110 | "time": time.time(), 111 | "gen_duration": gen_time, 112 | "gen_args": { 113 | "provider": "openrouter", 114 | "model": model, 115 | "modalities": ["image", "text"], 116 | }, 117 | } 118 | 119 | return output 120 | -------------------------------------------------------------------------------- /gas/utils/utils.py: -------------------------------------------------------------------------------- 1 | import concurrent.futures 2 | import functools 3 | import json 4 | import os 5 | import traceback 6 | 7 | import bittensor as bt 8 | 9 | 10 | def print_info(metagraph, hotkey, block, isMiner=True): 11 | uid = metagraph.hotkeys.index(hotkey) 12 | log = f"UID:{uid} | Block:{block} | Consensus:{metagraph.C[uid]} | " 13 | if isMiner: 14 | bt.logging.info( 15 | log 16 | + f"Stake:{metagraph.S[uid]} | Trust:{metagraph.T[uid]} | Incentive:{metagraph.I[uid]} | Emission:{metagraph.E[uid]}" 17 | ) 18 | return 19 | bt.logging.info(log + f"VTrust:{metagraph.Tv[uid]} | ") 20 | 21 | 22 | def fail_with_none(message: str = ""): 23 | def outer(func): 24 | def inner(*args, **kwargs): 25 | try: 26 | return func(*args, **kwargs) 27 | except Exception as e: 28 | bt.logging.error(message) 29 | bt.logging.error(str(e)) 30 | bt.logging.error(traceback.format_exc()) 31 | return None 32 | 33 | return inner 34 | 35 | return outer 36 | 37 | 38 | def on_block_interval(interval_attr_name): 39 | """ 40 | Decorator for methods that should only execute at specific block intervals. 41 | 42 | Args: 43 | interval_attr_name: String name of the config attribute that specifies the interval 44 | """ 45 | 46 | def decorator(func): 47 | @functools.wraps(func) 48 | async def wrapper(self, block, *args, **kwargs): 49 | if not self.initialization_complete: 50 | bt.logging.error(f"Block callbacks waiting for validator initialization to complete") 51 | return 52 | interval = getattr(self.config, interval_attr_name) 53 | if interval is None: 54 | bt.logging.error(f"No interval found for {interval_attr_name}") 55 | if ( 56 | block == 0 or block % interval == 0 57 | ): # Allow execution on block 0 for initialization 58 | return await func(self, block, *args, **kwargs) 59 | return None 60 | 61 | return wrapper 62 | 63 | return decorator 64 | 65 | 66 | class ExitContext: 67 | """ 68 | Using this as a class lets us pass this to other threads 69 | """ 70 | 71 | isExiting: bool = False 72 | 73 | def startExit(self, *_): 74 | if self.isExiting: 75 | exit() 76 | self.isExiting = True 77 | 78 | def __bool__(self): 79 | return self.isExiting 80 | 81 | 82 | def get_metadata(media_path): 83 | """Get metadata for a media file if it exists.""" 84 | base_path = os.path.splitext(media_path)[0] 85 | json_path = f"{base_path}.json" 86 | 87 | if os.path.exists(json_path): 88 | try: 89 | with open(json_path, "r") as f: 90 | return json.load(f) 91 | except json.JSONDecodeError: 92 | bt.logging.error(f"Warning: Could not parse JSON file: {json_path}") 93 | return {} 94 | return {} 95 | 96 | 97 | def get_file_modality(filepath: str) -> str: 98 | """ 99 | Determine the type of media file based on its extension. 100 | 101 | Args: 102 | filepath: Path to the media file 103 | 104 | Returns: 105 | "image", "video", or "file" based on the file extension 106 | """ 107 | ext = os.path.splitext(filepath)[1].lower() 108 | if ext in [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".webp"]: 109 | return "image" 110 | elif ext in [".mp4", ".avi", ".mov", ".webm", ".mkv", ".flv"]: 111 | return "video" 112 | else: 113 | return "file" 114 | 115 | 116 | def run_in_thread(func, timeout: int = 60): 117 | """Run a function in a thread with timeout""" 118 | with concurrent.futures.ThreadPoolExecutor() as executor: 119 | future = executor.submit(func) 120 | try: 121 | return future.result(timeout=timeout) 122 | except concurrent.futures.TimeoutError: 123 | raise TimeoutError(f"Function timed out after {timeout} seconds") 124 | -------------------------------------------------------------------------------- /docs/Discriminative-Mining.md: -------------------------------------------------------------------------------- 1 | # Discriminative Mining Guide 2 | 3 | ## Before You Proceed 4 | 5 | Follow the [Installation Guide](Installation.md) to set up your environment before proceeding with mining operations. 6 | 7 | ## Discriminative Mining Overview 8 | 9 | - Miners are tasked with training multiclass classifiers that discern between genuine and AI-generated content, and are rewarded based on their accuracy. 10 | - For each challenge, a miner's model is presented an image or video and is required to respond with a multiclass prediction [$p_{real}$, $p_{synthetic}$, $p_{semisynthetic}$] indicating whether the media is real, fully generated, or partially modified by AI. 11 | 12 | ## Model Preparation 13 | 14 | Discriminative miners need to prepare ONNX models for classification tasks and package them as zip files. You'll need: 15 | 16 | **📖 [How to Create ONNX Models](ONNX.md)** - Complete guide for creating compatible ONNX models 17 | 18 | - `image_detector.zip` - Zip file containing your image classification ONNX model 19 | - `video_detector.zip` - Zip file containing your video classification ONNX model 20 | 21 | Each zip file should contain the corresponding ONNX model file (`image_detector.onnx` or `video_detector.onnx`). 22 | 23 | ## Pushing Your Model 24 | 25 | First, activate the virtual environment: 26 | ```bash 27 | source .venv/bin/activate 28 | ``` 29 | 30 | Once you have your ONNX models packaged as zip files, push them to the network using the `push` command. You can upload models one at a time or both together: 31 | 32 | ```bash 33 | # Upload both models together 34 | gascli d push \ 35 | --image-model image_detector.zip \ 36 | --video-model video_detector.zip \ 37 | --wallet-name your_wallet_name \ 38 | --wallet-hotkey your_hotkey_name 39 | 40 | # Or upload just the image model 41 | gascli d push \ 42 | --image-model image_detector.zip \ 43 | --wallet-name your_wallet_name \ 44 | --wallet-hotkey your_hotkey_name 45 | 46 | # Or upload just the video model 47 | gascli d push \ 48 | --video-model video_detector.zip \ 49 | --wallet-name your_wallet_name \ 50 | --wallet-hotkey your_hotkey_name 51 | ``` 52 | 53 | ### Command Options 54 | 55 | The `push` command accepts several parameters: 56 | 57 | ```bash 58 | gascli d push \ 59 | --image-model image_detector.zip \ 60 | --video-model video_detector.zip \ 61 | --wallet-name your_wallet_name \ 62 | --wallet-hotkey your_hotkey_name \ 63 | --netuid 34 \ 64 | --chain-endpoint wss://test.finney.opentensor.ai:443/ \ 65 | --retry-delay 60 66 | ``` 67 | 68 | **Parameters:** 69 | - `--image-model`: Path to image detector zip file (optional, but at least one model required) 70 | - `--video-model`: Path to video detector zip file (optional, but at least one model required) 71 | - `--wallet-name`: Bittensor wallet name (default: "default") 72 | - `--wallet-hotkey`: Bittensor hotkey name (default: "default") 73 | - `--netuid`: Subnet UID (default: 34) 74 | - `--chain-endpoint`: Subtensor network endpoint (default: "wss://test.finney.opentensor.ai:443/") 75 | - `--retry-delay`: Retry delay in seconds (default: 60) 76 | 77 | 78 | ### Packaging Your Models 79 | 80 | Before pushing, you need to package your ONNX models into zip files. The zip format helps keep the system flexible should we expand to other model formats, or ones that require multiple files. Currently, each zip file should contain the corresponding ONNX model: 81 | 82 | ```bash 83 | # Package image model 84 | zip image_detector.zip image_detector.onnx 85 | 86 | # Package video model 87 | zip video_detector.zip video_detector.onnx 88 | ``` 89 | 90 | ### What Happens During Push 91 | 92 | 1. **Model Validation**: The system checks that the zip files are present and valid 93 | 2. **Model Upload**: Your model zip files are uploaded to the cloud inference system 94 | 3. **Blockchain Registration**: Model metadata is registered on the Bittensor blockchain 95 | 4. **Verification**: The system verifies the registration was successful 96 | 97 | ### Getting Help 98 | 99 | ```bash 100 | gascli discriminator --help # Miner help 101 | gascli d push --help # Push command help 102 | ``` 103 | 104 | **Note**: Remember to activate the virtual environment first with `source .venv/bin/activate` before running any `gascli` commands. -------------------------------------------------------------------------------- /gas/evaluation/miner_type_tracker.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, List, Optional, Any 2 | import time 3 | 4 | import bittensor as bt 5 | import aiohttp 6 | import asyncio 7 | 8 | from gas.protocol.validator_requests import get_miner_type 9 | from gas.types import MinerType 10 | 11 | 12 | class MinerTypeTracker: 13 | """Tracks miner types and manages periodic updates""" 14 | 15 | TYPE_CACHE_TTL = 300 # Only re-query miners every 5 minutes 16 | 17 | def __init__(self, config, wallet, metagraph, subtensor): 18 | self.config = config 19 | self.wallet = wallet 20 | self.metagraph = metagraph 21 | self.subtensor = subtensor 22 | 23 | self.miner_types: Dict[int, MinerType] = {} 24 | self.last_update: Dict[int, float] = {} 25 | self._last_full_update: float = 0 26 | 27 | async def initialize_miner_types(self): 28 | """Initialize miner types for all registered miners""" 29 | bt.logging.debug("Initializing miner types for all registered miners...") 30 | all_miners = list(range(self.metagraph.n.item())) 31 | await self.update_miner_types(all_miners) 32 | bt.logging.debug(f"Initialized miner types for {len(self.miner_types)} miners") 33 | 34 | async def update_miner_types(self, miner_uids: Optional[List[int]] = None, force: bool = False): 35 | """Update miner types for specified miners, skipping recently queried ones""" 36 | current_time = time.time() 37 | 38 | if miner_uids is None: 39 | # Skip full update if we did one recently 40 | if not force and (current_time - self._last_full_update) < self.TYPE_CACHE_TTL: 41 | bt.logging.trace("Skipping miner type update - cache still fresh") 42 | return 43 | miner_uids = list(range(self.metagraph.n)) 44 | self._last_full_update = current_time 45 | 46 | # Filter to only stale entries (not queried within TTL) 47 | if not force: 48 | stale_uids = [ 49 | uid for uid in miner_uids 50 | if (current_time - self.last_update.get(uid, 0)) >= self.TYPE_CACHE_TTL 51 | ] 52 | if not stale_uids: 53 | bt.logging.trace(f"All {len(miner_uids)} miners have fresh type cache") 54 | return 55 | miner_uids = stale_uids 56 | 57 | bt.logging.debug(f"Querying miner types for {len(miner_uids)} UIDs") 58 | async with aiohttp.ClientSession() as session: 59 | responses = await asyncio.gather( 60 | *[ 61 | get_miner_type( 62 | uid, 63 | self.metagraph.axons[uid], 64 | session, 65 | self.wallet.hotkey, 66 | self.config.neuron.miner_total_timeout, 67 | ) 68 | for uid in miner_uids 69 | ], 70 | return_exceptions=True, 71 | ) 72 | 73 | current_time = time.time() 74 | for uid, response in zip(miner_uids, responses): 75 | # We assume a miner is a discriminator (which does not run any endpoint/hardware) 76 | # until we successfully get a response from a generator get_miner_info endpoint 77 | miner_type = MinerType.DISCRIMINATOR 78 | 79 | if isinstance(response, Exception): 80 | continue 81 | 82 | miner_type_str = response.get("miner_type", "") or "" 83 | if miner_type_str.lower() == "generator": 84 | miner_type = MinerType.GENERATOR 85 | 86 | if miner_type: 87 | old_type = self.miner_types.get(uid) 88 | self.miner_types[uid] = miner_type 89 | self.last_update[uid] = current_time 90 | 91 | if old_type != miner_type: 92 | bt.logging.trace(f"UID {uid}: {old_type} -> {miner_type}") 93 | else: 94 | bt.logging.trace(f"UID {uid}: {miner_type}") 95 | 96 | def get_miner_type(self, uid: int) -> Optional[MinerType]: 97 | return self.miner_types.get(uid) 98 | 99 | def get_miners_by_type(self, miner_type: MinerType) -> List[int]: 100 | return [uid for uid, mt in self.miner_types.items() if mt == miner_type] 101 | -------------------------------------------------------------------------------- /gas/cache/types.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass, asdict 2 | from typing import Dict, Any, Optional, List 3 | import numpy as np 4 | import time 5 | import uuid 6 | 7 | from gas.types import Modality, MediaType, SourceType 8 | 9 | 10 | @dataclass 11 | class Media: 12 | """Media representation for db writes""" 13 | modality: Modality 14 | media_type: MediaType 15 | media_content: Any # PIL Image, video frames, etc. 16 | format: str # "JPEG", "PNG", "MP4", etc. 17 | prompt_id: Optional[str] = None # None for dataset media, str for generated media 18 | 19 | # synthetic & semisynthetic 20 | model_name: Optional[str] = None 21 | generation_args: Optional[Dict[str, Any]] = None 22 | 23 | # semisynthetic 24 | original_media_id: Optional[str] = None 25 | mask_content: Optional[np.ndarray] = None 26 | 27 | # Common metadata 28 | metadata: Optional[Dict[str, Any]] = None 29 | 30 | 31 | @dataclass 32 | class PromptEntry: 33 | """Represents a prompt entry in the database""" 34 | 35 | id: str 36 | content: str 37 | content_type: str # "prompt" or "search_query" 38 | created_at: float 39 | used_count: int = 0 40 | last_used: Optional[float] = None 41 | source_media_id: Optional[str] = None # ID of the media that was used to generate this prompt 42 | 43 | def to_dict(self) -> Dict[str, Any]: 44 | return asdict(self) 45 | 46 | 47 | @dataclass 48 | class MediaEntry: 49 | """Represents a media entry linked to a prompt - database record for stored media""" 50 | 51 | id: str 52 | prompt_id: str 53 | file_path: str 54 | modality: Modality 55 | media_type: MediaType 56 | source_type: SourceType = SourceType.GENERATED # "scraper", "dataset", "generated" 57 | 58 | # For synthetic media (generated content) 59 | model_name: Optional[str] = None 60 | generation_args: Optional[Dict[str, Any]] = None 61 | 62 | # For real media (downloaded content) 63 | download_url: Optional[str] = None 64 | scraper_name: Optional[str] = None 65 | 66 | # For dataset media 67 | dataset_name: Optional[str] = None 68 | dataset_source_file: Optional[str] = None 69 | dataset_index: Optional[str] = None 70 | 71 | # For miner media 72 | uid: Optional[int] = None 73 | hotkey: Optional[str] = None 74 | verified: Optional[bool] = False 75 | failed_verification: Optional[bool] = False 76 | 77 | uploaded: Optional[bool] = False 78 | rewarded: Optional[bool] = False 79 | 80 | prompt_content: Optional[str] = None # for hf uploads 81 | 82 | # Duplicate detection 83 | perceptual_hash: Optional[str] = None # pHash for duplicate detection 84 | 85 | # C2PA content credentials 86 | c2pa_verified: Optional[bool] = False # C2PA validation passed 87 | c2pa_issuer: Optional[str] = None # Issuer name if C2PA verified 88 | 89 | # Common fields 90 | created_at: float = None 91 | resolution: Optional[tuple[int, int]] = None # (width, height) 92 | file_size: Optional[int] = None # in bytes 93 | format: Optional[str] = None # File format (e.g., "PNG", "JPEG", "MP4") 94 | 95 | def __post_init__(self): 96 | if self.created_at is None: 97 | self.created_at = time.time() 98 | 99 | def to_dict(self) -> Dict[str, Any]: 100 | data = asdict(self) 101 | if isinstance(data.get("modality"), Modality): 102 | data["modality"] = data["modality"].value 103 | if isinstance(data.get("media_type"), MediaType): 104 | data["media_type"] = data["media_type"].value 105 | if isinstance(data.get("source_type"), SourceType): 106 | data["source_type"] = data["source_type"].value 107 | return data 108 | 109 | 110 | class VerificationResult: 111 | """Miner-generated data verification""" 112 | def __init__( 113 | self, 114 | media_entry: MediaEntry, 115 | original_prompt: Optional[str] = None, 116 | generated_caption: Optional[str] = None, 117 | verification_score: Optional[Dict[str, Any]] = None, 118 | passed: bool = False, 119 | ): 120 | self.media_entry = media_entry 121 | self.original_prompt = original_prompt 122 | self.generated_caption = generated_caption 123 | self.verification_score = verification_score 124 | self.passed = passed 125 | -------------------------------------------------------------------------------- /gas/utils/wandb_utils.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os 3 | import shutil 4 | import time 5 | 6 | import bittensor as bt 7 | import wandb 8 | from gas import __version__ 9 | 10 | 11 | def init_wandb( 12 | config: bt.config, process: str, uid: int, hotkey: bt.Keypair, wandb_dir: str = None 13 | ) -> wandb.run: 14 | """ 15 | Initialize a Weights & Biases run. 16 | 17 | Args: 18 | config: Bittensor config object 19 | process: Valid options are 'validator', 'data-generator', 'media-store' 20 | uid: Validator uid 21 | hotkey: Bittensor keypair for signing the run 22 | wandb_dir: Optional directory for wandb files 23 | 24 | Returns: 25 | wandb.run: The initialized wandb run, or None if initialization fails 26 | """ 27 | project = f"subnet-{config.netuid}-{process}" 28 | run_name = f"{process}-{uid}-{__version__}" 29 | config.run_name = run_name 30 | config.uid = uid 31 | config.hotkey = hotkey.ss58_address 32 | config.version = __version__ 33 | 34 | bt.logging.info(f"Initializing wandb run in '{config.wandb.entity}/{project}'") 35 | 36 | try: 37 | run = wandb.init( 38 | name=run_name, 39 | project=project, 40 | entity=config.wandb.entity, 41 | config=config, 42 | dir=wandb_dir if wandb_dir else config.full_path, 43 | reinit=True, 44 | ) 45 | except wandb.UsageError as e: 46 | bt.logging.warning(e) 47 | bt.logging.warning("Did you run wandb login?") 48 | return 49 | 50 | # sign the run to prove it's from this hotkey 51 | signature = hotkey.sign(run.id.encode()).hex() 52 | config.signature = signature 53 | wandb.config.update(config, allow_val_change=True) 54 | 55 | bt.logging.success(f"Started wandb run {run_name}") 56 | return run 57 | 58 | 59 | def clean_wandb_cache(wandb_dir, hours=1): 60 | """ 61 | Cleans wandb runs except recent ones and latest-run. 62 | 63 | Args: 64 | wandb_dir: Directory containing wandb run files 65 | hours: Number of hours to keep runs for (default: 1) 66 | """ 67 | if not wandb_dir.endswith("wandb"): 68 | wandb_dir = os.path.join(wandb_dir, "wandb") 69 | 70 | if not os.path.exists(wandb_dir): 71 | bt.logging.warning(f"W&B directory not found: {wandb_dir}") 72 | return 73 | 74 | bt.logging.info(f"Attempting to clean wandb cache at {wandb_dir}") 75 | run_dirs = [ 76 | d for d in glob.glob(os.path.join(wandb_dir, "run-*")) if os.path.isdir(d) 77 | ] 78 | 79 | if not run_dirs: 80 | bt.logging.info("No W&B runs found.") 81 | return 82 | 83 | # Keep recent runs 84 | current_time = time.time() 85 | cutoff_time = current_time - (hours * 3600) 86 | recent_runs = [d for d in run_dirs if os.path.getmtime(d) > cutoff_time] 87 | 88 | # Preserve latest-run target 89 | latest_run_link = os.path.join(wandb_dir, "latest-run") 90 | if os.path.exists(latest_run_link) and os.path.isdir(latest_run_link): 91 | try: 92 | latest_run_target = os.path.realpath(latest_run_link) 93 | if latest_run_target not in recent_runs and latest_run_target in run_dirs: 94 | recent_runs.append(latest_run_target) 95 | bt.logging.debug( 96 | f"Preserving latest-run: {os.path.basename(latest_run_target)}" 97 | ) 98 | except Exception as e: 99 | bt.logging.warning(f"Error with latest-run: {e}") 100 | 101 | bt.logging.info( 102 | f"Keeping {len(recent_runs)} runs (modified in last {hours} hours):" 103 | ) 104 | for run in recent_runs: 105 | bt.logging.info(f" - {os.path.basename(run)}") 106 | 107 | runs_removed = 0 108 | space_freed = 0 109 | for run_dir in run_dirs: 110 | if run_dir not in recent_runs: 111 | try: 112 | dir_size = sum( 113 | os.path.getsize(os.path.join(dirpath, filename)) 114 | for dirpath, _, filenames in os.walk(run_dir) 115 | for filename in filenames 116 | ) 117 | space_freed += dir_size 118 | shutil.rmtree(run_dir) 119 | runs_removed += 1 120 | except Exception as e: 121 | bt.logging.warning(f"Error removing {run_dir}: {e}") 122 | 123 | bt.logging.info( 124 | f"Cleaned {runs_removed} runs, freed {space_freed / (1024*1024):.2f} MB" 125 | ) 126 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | BitMind Logo 3 | 4 |

GAS
Generative Adversarial Subnet

5 |

Bittensor SN34

6 |

7 | ⛏️ Mining · 8 | 🛡️ Validating · 9 | 💰 Incentives · 10 | 🏆 Leaderboard 11 |

12 |

13 | 🤗 GAS-Station · 14 | 🌐 Apps 15 |

16 |
17 | 18 | ## About GAS 19 |
20 | Fake content is evolving fast. Staying ahead demands relentless innovation.

21 |
22 | 23 | **GAS (Generative Adversarial Subnet)** is a Bittensor subnet inspired by Generative Adversarial Networks (GANs). Detectors and generators compete in a dynamic loop: detectors sharpen their ability to spot synthetic media, while generators push to create more convincing fakes. This adversarial process drives cutting-edge detection tools and continuously generates the training data needed to sustain progress. 24 | 25 | Unlike static AI safety solutions, GAS thrives on open, incentivized competition, ensuring detectors evolve as fast as the threats they face. 26 | 27 | 28 | ## Quick Start 29 | 30 | ### Installation 31 | 32 | ```bash 33 | git clone 34 | cd GAS 35 | ./install.sh 36 | ``` 37 | 38 | **Options:** 39 | - `./install.sh --no-system-deps` - Skip system dependency installation (intended for discriminative miners) 40 | 41 | ### Using gascli 42 | ```bash 43 | # Activate virtual environment to use gascli 44 | source .venv/bin/activate 45 | 46 | # Show available commands 47 | gascli --help 48 | 49 | # Validators: Start or restart validator services 50 | gascli validator start 51 | 52 | # Miners: Start or restart generative miner 53 | gascli generator start 54 | 55 | # Miners: Push discriminator models (both at once) 56 | gascli discriminator push --image-model image_detector.zip --video-model video_detector.zip --wallet-name default --wallet-hotkey default 57 | 58 | # Or push one model at a time 59 | gascli d push --image-model image_detector.zip 60 | gascli d push --video-model video_detector.zip 61 | ``` 62 | 63 | **Available Aliases:** 64 | - `validator` → `vali`, `v` 65 | - `discriminator` → `d` 66 | - `generator` → `g` 67 | 68 | 69 | ### Not using gascli 70 | ```bash 71 | # Validators: Start or restart validator services 72 | # (Does not require virtualenv activation) 73 | pm2 start validator.config.js 74 | 75 | # Miners: Start or restart generative miner 76 | pm2 start gen_miner.config.js 77 | 78 | # Miners: Push discriminator models 79 | source .venv/bin/activate 80 | python neurons/discriminator/push_model.py --image-model image_detector.zip --video-model video_detector.zip --wallet-name default --wallet-hotkey default 81 | ``` 82 | For detailed installation and usage instructions, see [Installation Guide](docs/Installation.md). 83 | 84 | 85 | ## Core Components 86 | 87 | > This documentation assumes basic familiarity with [Bittensor concepts](https://docs.bittensor.com/learn/bittensor-building-blocks). 88 | 89 | #### Discriminative Miners [[docs](docs/Discriminative-Mining.md)] 90 | Discriminative miners submit detection models for evaluation against a wide variety of real and synthetic media and are rewarded based on their accuracy. This differs from previous versions of SN34, where discriminative miners hosted hardware to serve both validator challenges and organic API traffic. This both significantly reduces the capital required to mine, and allows the subnet to more reliably identiy unique models and reward novel contributions proportionally to their accuracy rather than the speed of their registration script. 91 | 92 | 93 | #### Generative Miners [[docs](docs/Generative-Mining.md)] 94 | 95 | Generative miners generate and modify media according to prompts generated by validators, and are rewarded based on their ability to pass validation checks and fool discrimintive miners. 96 | 97 | #### Validators [[docs](docs/Validating.md)] 98 | Validators are responsible for challenging and scoring both miner types. Generative miners are sent prompts, and their returned synthetic media are validated to mitigate gaming and incentivize high quality results. Discriminative miners are continually evaluated against a mix of data from generative miners, real world data, and data generated locally on the validator. 99 | 100 | 101 | ## Subnet Architecture 102 | ![Subnet Architecture](docs/static/GAS-Architecture-Simple.png) 103 | 104 | ## Community 105 | 106 |

107 | 108 | Join us on Discord 109 | 110 |

111 | 112 | ## Contributing 113 | 114 | Contributions are welcome and can be made via a pull request to the `testnet` branch. 115 | 116 | [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/BitMind-AI/bitmind-subnet) 117 | -------------------------------------------------------------------------------- /tests/generator/stabilityai_service.py: -------------------------------------------------------------------------------- 1 | import os 2 | import traceback 3 | from PIL import Image 4 | import io 5 | import time 6 | 7 | from neurons.generator.services.stabilityai_service import StabilityAIService, Models 8 | from neurons.generator.task_manager import TaskManager 9 | from gas.verification.c2pa_verification import verify_c2pa 10 | 11 | # Set API key if one isn't already set 12 | os.environ.setdefault( 13 | "STABILITY_API_KEY", 14 | "sk-YOUR-API-KEY" # replace with your test key 15 | ) 16 | 17 | def save_image(img_bytes, filename): 18 | os.makedirs("outputs", exist_ok=True) 19 | out_path = f"outputs/{filename}" 20 | with open(out_path, "wb") as f: 21 | f.write(img_bytes) 22 | return out_path 23 | 24 | def validate_image(img_bytes): 25 | try: 26 | Image.open(io.BytesIO(img_bytes)).verify() 27 | return True 28 | except Exception: 29 | return False 30 | 31 | 32 | def run_model_test(service, manager, model): 33 | print(f"\n=== Running generation test for model: {model} ===") 34 | 35 | # Use TaskManager.create_task() exactly like in production 36 | task_id = manager.create_task( 37 | modality="image", 38 | prompt="A neon cyberpunk city with flying cars", 39 | parameters={ 40 | "model": model, 41 | "format": "png", 42 | "aspect_ratio": "16:9", 43 | "seed": 777, 44 | "negative_prompt": "low quality, blurry" 45 | }, 46 | webhook_url=None, 47 | signed_by="test-suite" 48 | ) 49 | 50 | task = manager.get_task(task_id) 51 | 52 | try: 53 | start_time = time.time() 54 | result = service.process(task) 55 | elapsed = time.time() - start_time 56 | 57 | img_bytes = result["data"] 58 | meta = result["metadata"] 59 | 60 | print(f"✔ Generated image in {elapsed:.2f}s ({len(img_bytes)/1024:.1f} KB)") 61 | print(f"✔ Metadata keys: {list(meta.keys())}") 62 | 63 | # Save output 64 | out = save_image(img_bytes, f"{model.replace('.', '_')}.png") 65 | print(f"✔ Saved to {out}") 66 | 67 | # Validate image integrity 68 | assert validate_image(img_bytes), "Image failed Pillow validation" 69 | 70 | # After generating with Runway 71 | result = verify_c2pa(img_bytes) 72 | if result.verified and result.is_trusted_issuer: 73 | print(f"✅ C2PA verified: {result.issuer}") 74 | else: 75 | print(f"❌ C2PA failed: {result.error}") 76 | 77 | # Validate metadata 78 | assert meta["model"] == model 79 | assert meta["provider"] == "stability.ai" 80 | assert meta["format"] in ("PNG", "JPEG", "WEBP") 81 | assert meta["generation_time"] > 0 82 | 83 | print(f"=== Model {model} PASSED ===") 84 | 85 | except Exception: 86 | print(f"=== Model {model} FAILED ===") 87 | print(traceback.format_exc()) 88 | 89 | 90 | def test_invalid_api_key(): 91 | print("\n=== Testing invalid API key ===") 92 | os.environ["STABILITY_API_KEY"] = "invalid-key" 93 | 94 | service = StabilityAIService() 95 | manager = TaskManager() 96 | 97 | task_id = manager.create_task( 98 | modality="image", 99 | prompt="test prompt", 100 | parameters={"model": Models.SD35_MEDIUM}, 101 | webhook_url=None, 102 | signed_by="test" 103 | ) 104 | task = manager.get_task(task_id) 105 | 106 | try: 107 | service.process(task) 108 | raise AssertionError("❌ Should have failed with invalid API key!") 109 | except Exception as e: 110 | print(f"✔ Correctly failed: {e}") 111 | 112 | 113 | def test_invalid_format(service, manager): 114 | print("\n=== Testing invalid format fallback ===") 115 | 116 | task_id = manager.create_task( 117 | modality="image", 118 | prompt="test prompt", 119 | parameters={"model": Models.SD35_MEDIUM, "format": "tiff"}, 120 | webhook_url=None, 121 | signed_by="test" 122 | ) 123 | task = manager.get_task(task_id) 124 | 125 | result = service.process(task) 126 | assert result["metadata"]["format"] == "PNG" 127 | print("✔ Invalid format 'tiff' auto-corrected to PNG") 128 | 129 | 130 | def run_full_test_suite(): 131 | print("\n========== StabilityAI Full Test Suite ==========\n") 132 | 133 | # Restore real key 134 | os.environ["STABILITY_API_KEY"] = os.getenv("STABILITY_API_KEY") 135 | 136 | service = StabilityAIService() 137 | manager = TaskManager() 138 | 139 | if not service.is_available(): 140 | print("❌ API key missing — cannot run tests") 141 | return 142 | 143 | # Test all models 144 | for model in [ 145 | Models.SD35_MEDIUM, 146 | Models.SD35_LARGE, 147 | Models.ULTRA, 148 | Models.CORE 149 | ]: 150 | run_model_test(service, manager, model) 151 | 152 | # Negative tests 153 | test_invalid_format(service, manager) 154 | test_invalid_api_key() 155 | 156 | print("\n========== All Tests Completed ==========\n") 157 | 158 | 159 | if __name__ == "__main__": 160 | run_full_test_suite() 161 | -------------------------------------------------------------------------------- /gas/utils/metagraph.py: -------------------------------------------------------------------------------- 1 | import time 2 | import asyncio 3 | from typing import Callable, List, Tuple 4 | import numpy as np 5 | import bittensor as bt 6 | from bittensor.utils.weight_utils import process_weights_for_netuid 7 | from async_substrate_interface import AsyncSubstrateInterface 8 | 9 | from gas.utils import fail_with_none 10 | 11 | 12 | def get_miner_uids( 13 | metagraph: "bt.metagraph", self_uid: int, vpermit_tao_limit: int 14 | ) -> List[int]: 15 | available_uids = [] 16 | for uid in range(int(metagraph.n.item())): 17 | if uid == self_uid: 18 | continue 19 | 20 | # Filter validator permit > 1024 stake. 21 | if metagraph.validator_permit[uid]: 22 | if metagraph.S[uid] > vpermit_tao_limit: 23 | continue 24 | 25 | available_uids.append(uid) 26 | 27 | return available_uids 28 | 29 | 30 | def create_set_weights(version: int, netuid: int): 31 | @fail_with_none("Failed setting weights") 32 | def set_weights( 33 | wallet: "bt.wallet", 34 | metagraph: "bt.metagraph", 35 | subtensor: "bt.subtensor", 36 | weights: Tuple[List[int], List[float]], 37 | ): 38 | uids, raw_weights = weights 39 | if not len(uids): 40 | bt.logging.info("No UIDS to score") 41 | return 42 | 43 | # Set the weights on chain via our subtensor connection. 44 | ( 45 | processed_weight_uids, 46 | processed_weights, 47 | ) = process_weights_for_netuid( 48 | uids=np.asarray(uids), 49 | weights=np.asarray(raw_weights), 50 | netuid=netuid, 51 | subtensor=subtensor, 52 | metagraph=metagraph, 53 | ) 54 | 55 | bt.logging.info("Setting Weights: " + str(processed_weights)) 56 | bt.logging.info("Weight Uids: " + str(processed_weight_uids)) 57 | for _ in range(3): 58 | result, message = subtensor.set_weights( 59 | wallet=wallet, 60 | netuid=netuid, 61 | uids=processed_weight_uids, # type: ignore 62 | weights=processed_weights, 63 | wait_for_finalization=False, 64 | wait_for_inclusion=False, 65 | version_key=version, 66 | max_retries=1, 67 | ) 68 | if result is True: 69 | bt.logging.success("set_weights on chain successfully!") 70 | break 71 | else: 72 | bt.logging.error(f"set_weights failed {message}") 73 | time.sleep(15) 74 | 75 | return set_weights 76 | 77 | 78 | def create_async_subscription_handler(callback: Callable): 79 | """Create async subscription handler - simple version.""" 80 | async def handler(obj): 81 | try: 82 | # Extract block number just like sync version 83 | block_number = obj["header"]["number"] 84 | await callback(block_number) 85 | except Exception as e: 86 | bt.logging.error(f"Error in async substrate block callback: {e}") 87 | 88 | return handler 89 | 90 | 91 | async def start_async_subscription(substrate, callback: Callable): 92 | """Start async block subscription - mirroring the sync version.""" 93 | return await substrate.subscribe_block_headers( 94 | create_async_subscription_handler(callback) 95 | ) 96 | 97 | 98 | class SubstrateConnectionManager: 99 | """Async substrate connection manager with auto-reconnection.""" 100 | 101 | def __init__(self, url: str, ss58_format: int, type_registry: dict): 102 | self.url = url 103 | self.ss58_format = ss58_format 104 | self.type_registry = type_registry 105 | self.running = False 106 | self.task = None 107 | 108 | async def start_subscription(self, callback: Callable): 109 | """Start subscription with auto-reconnect.""" 110 | self.running = True 111 | 112 | while self.running: 113 | try: 114 | bt.logging.info(f"Connecting to async substrate: {self.url}") 115 | 116 | substrate = AsyncSubstrateInterface( 117 | url=self.url, 118 | ss58_format=self.ss58_format, 119 | type_registry=self.type_registry, 120 | ) 121 | 122 | async with substrate: 123 | bt.logging.info("Starting async block subscription") 124 | await start_async_subscription(substrate, callback) 125 | 126 | except Exception as e: 127 | bt.logging.error(f"Async substrate failed: {e}") 128 | if self.running: 129 | bt.logging.info("Reconnecting in 5 seconds...") 130 | await asyncio.sleep(5) 131 | 132 | def start_subscription_task(self, callback: Callable): 133 | """Start as background task.""" 134 | self.task = asyncio.create_task(self.start_subscription(callback)) 135 | return self.task 136 | 137 | def stop(self): 138 | """Stop subscription.""" 139 | self.running = False 140 | if self.task: 141 | self.task.cancel() 142 | 143 | -------------------------------------------------------------------------------- /gen_miner.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const os = require('os'); 3 | 4 | // Helper functions 5 | function getPythonInterpreter() { 6 | const projectRoot = __dirname; 7 | const venvPython = path.join(projectRoot, '.venv', 'bin', 'python'); 8 | const fs = require('fs'); 9 | return fs.existsSync(venvPython) ? venvPython : 'python3'; 10 | } 11 | 12 | function getNetworkSettings(chainEndpoint) { 13 | if (chainEndpoint.includes('test')) return 379; 14 | if (chainEndpoint.includes('finney')) return 34; 15 | return null; 16 | } 17 | 18 | function getLogParam(loglevel) { 19 | switch (loglevel) { 20 | case 'trace': return '--logging.trace'; 21 | case 'debug': return '--logging.debug'; 22 | default: return '--logging.info'; 23 | } 24 | } 25 | 26 | function getAutoUpdateParam(autoUpdate) { 27 | return autoUpdate === 'true' ? '' : '--autoupdate-off'; 28 | } 29 | 30 | // Dynamically load environment variables from .env.gen_miner 31 | const envPath = path.resolve(__dirname, '.env.gen_miner'); 32 | const envConfig = require('dotenv').config({ path: envPath }); 33 | const envFileVars = envConfig.parsed || {}; 34 | 35 | // Apply to process.env for compatibility with existing config logic 36 | Object.assign(process.env, envFileVars); 37 | const config = { 38 | // Wallet 39 | walletName: process.env.BT_WALLET_NAME || 'default', 40 | walletHotkey: process.env.BT_WALLET_HOTKEY || 'default', 41 | 42 | // Network 43 | chainEndpoint: process.env.BT_CHAIN_ENDPOINT || 'wss://test.finney.opentensor.ai:443', 44 | netuid: process.env.BT_NETUID || '379', 45 | 46 | // Axon Configuration 47 | axonPort: process.env.BT_AXON_PORT || '8093', 48 | axonIp: process.env.BT_AXON_IP || '0.0.0.0', 49 | axonExternalIp: process.env.BT_AXON_EXTERNAL_IP || 'auto', 50 | 51 | // Device 52 | device: process.env.MINER_DEVICE || 'auto', 53 | 54 | // Logging 55 | loglevel: process.env.BT_LOGGING_LEVEL || 'info', 56 | 57 | // Features 58 | autoUpdate: process.env.AUTO_UPDATE || 'false', 59 | 60 | // Miner-specific configuration 61 | outputDir: process.env.MINER_OUTPUT_DIR || '/tmp/generated_content', 62 | maxConcurrentTasks: process.env.MINER_MAX_CONCURRENT_TASKS || '5', 63 | workerThreads: process.env.MINER_WORKER_THREADS || '2', 64 | taskTimeout: process.env.MINER_TASK_TIMEOUT || '300', 65 | 66 | // Force permit setting 67 | noForceValidatorPermit: process.env.MINER_NO_FORCE_VALIDATOR_PERMIT === 'true', 68 | }; 69 | 70 | // Determine netuid (override from env or derive from chain endpoint) 71 | const netuid = process.env.BT_NETUID || getNetworkSettings(config.chainEndpoint); 72 | 73 | // Build command parameters 74 | const logParam = getLogParam(config.loglevel); 75 | const autoUpdateParam = getAutoUpdateParam(config.autoUpdate); 76 | const pythonInterpreter = getPythonInterpreter(); 77 | 78 | // Project paths 79 | const projectRoot = __dirname; 80 | const minerScript = path.join(projectRoot, 'neurons', 'generator', 'miner.py'); 81 | 82 | // Allow optional override of HF cache dir via env. Must be resolved before any Python starts. 83 | const HF_HOME_RESOLVED = process.env.HF_HOME 84 | || process.env.HUGGINGFACE_HOME 85 | || process.env.HUGGINGFACE_CACHE_DIR 86 | || path.join(os.homedir(), '.cache', 'huggingface'); 87 | 88 | // Build dynamic environment from .env.gen_miner file 89 | // Adding new API keys to .env.gen_miner makes them available 90 | // to the miner without needing to modify this config file 91 | const DYNAMIC_ENV = { 92 | HF_HOME: HF_HOME_RESOLVED, 93 | HF_HUB_DISABLE_TELEMETRY: '1', 94 | ...envFileVars, 95 | }; 96 | 97 | // Build miner arguments 98 | const minerArgs = [ 99 | '--wallet.name', config.walletName, 100 | '--wallet.hotkey', config.walletHotkey, 101 | '--netuid', netuid.toString(), 102 | '--subtensor.chain_endpoint', config.chainEndpoint, 103 | '--axon.port', config.axonPort, 104 | '--axon.ip', config.axonIp, 105 | '--device', config.device, 106 | '--miner.output-dir', config.outputDir, 107 | '--miner.max-concurrent-tasks', config.maxConcurrentTasks, 108 | '--miner.worker-threads', config.workerThreads, 109 | '--miner.task-timeout', config.taskTimeout, 110 | logParam, 111 | ]; 112 | 113 | // Add conditional arguments 114 | if (autoUpdateParam) { 115 | minerArgs.push(autoUpdateParam); 116 | } 117 | 118 | if (config.noForceValidatorPermit) { 119 | minerArgs.push('--no-force-validator-permit'); 120 | } 121 | 122 | if (config.axonExternalIp && config.axonExternalIp !== 'auto') { 123 | minerArgs.push('--axon.external_ip', config.axonExternalIp); 124 | } 125 | 126 | // PM2 Apps configuration 127 | const apps = [ 128 | { 129 | name: 'bitmind-generative-miner', 130 | script: minerScript, 131 | interpreter: pythonInterpreter, 132 | args: minerArgs.join(' '), 133 | env: { 134 | ...DYNAMIC_ENV, 135 | }, 136 | watch: false, 137 | instances: 1, 138 | autorestart: true, 139 | max_restarts: 10, 140 | min_uptime: '10s', 141 | restart_delay: 4000, 142 | error_file: path.join(config.outputDir, 'logs', 'miner-error.log'), 143 | out_file: path.join(config.outputDir, 'logs', 'miner-out.log'), 144 | log_file: path.join(config.outputDir, 'logs', 'miner-combined.log'), 145 | time: true, 146 | } 147 | ]; 148 | 149 | module.exports = { 150 | apps, 151 | }; 152 | 153 | -------------------------------------------------------------------------------- /gas/generation/model_registry.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Dict, Union, Any, List 2 | import random 3 | 4 | from gas.types import ModelConfig, ModelTask, MediaType, Modality 5 | 6 | 7 | class ModelRegistry: 8 | """ 9 | Registry for managing generative models. 10 | """ 11 | 12 | def __init__(self): 13 | self.models: Dict[str, ModelConfig] = {} 14 | 15 | def register(self, model_config: ModelConfig) -> None: 16 | self.models[model_config.path] = model_config 17 | 18 | def register_all(self, model_configs: List[ModelConfig]) -> None: 19 | for config in model_configs: 20 | self.register(config) 21 | 22 | def get_model(self, path: str) -> Optional[ModelConfig]: 23 | return self.models.get(path) 24 | 25 | def get_all_models(self) -> Dict[str, ModelConfig]: 26 | return self.models.copy() 27 | 28 | def get_models_by_task(self, tasks) -> Dict[str, ModelConfig]: 29 | if tasks is None: 30 | return self.get_all_models() 31 | 32 | if not isinstance(tasks, list): 33 | tasks = [tasks] 34 | 35 | if not isinstance(tasks[0], ModelTask): 36 | tasks = [ModelTask(t) for t in tasks if t in ModelTask.__members__] 37 | 38 | return { 39 | path: config for path, config in self.models.items() if config.task in tasks 40 | } 41 | 42 | def get_models_by_tag(self, tag: str) -> Dict[str, ModelConfig]: 43 | return { 44 | path: config for path, config in self.models.items() if tag in config.tags 45 | } 46 | 47 | def get_model_names_by_task(self, tasks: Union[ModelTask, List[ModelTask], str, List[str]]) -> List[str]: 48 | if "all" in tasks: 49 | return self.get_interleaved_model_names() 50 | 51 | return list(self.get_models_by_task(tasks).keys()) 52 | 53 | @property 54 | def t2i_models(self) -> Dict[str, ModelConfig]: 55 | return self.get_models_by_task(ModelTask.TEXT_TO_IMAGE) 56 | 57 | @property 58 | def t2v_models(self) -> Dict[str, ModelConfig]: 59 | return self.get_models_by_task(ModelTask.TEXT_TO_VIDEO) 60 | 61 | @property 62 | def i2i_models(self) -> Dict[str, ModelConfig]: 63 | return self.get_models_by_task(ModelTask.IMAGE_TO_IMAGE) 64 | 65 | @property 66 | def i2v_models(self) -> List[str]: 67 | return self.get_models_by_task(ModelTask.IMAGE_TO_VIDEO) 68 | 69 | @property 70 | def t2i_model_names(self) -> List[str]: 71 | return list(self.t2i_models.keys()) 72 | 73 | @property 74 | def t2v_model_names(self) -> List[str]: 75 | return list(self.t2v_models.keys()) 76 | 77 | @property 78 | def i2i_model_names(self) -> List[str]: 79 | return list(self.i2i_models.keys()) 80 | 81 | @property 82 | def i2v_model_names(self) -> List[str]: 83 | return list(self.i2v_models.keys()) 84 | 85 | @property 86 | def model_names(self) -> List[str]: 87 | return list(self.models.keys()) 88 | 89 | def select_random_model(self, task: Optional[Union[ModelTask, str]] = None) -> str: 90 | if isinstance(task, str): 91 | task = ModelTask(task.lower()) 92 | 93 | if task is None: 94 | task = random.choice(list(ModelTask)) 95 | 96 | model_names = self.get_model_names_by_task(task) 97 | if not model_names: 98 | raise ValueError(f"No models available for task: {task}") 99 | 100 | return random.choice(model_names) 101 | 102 | def get_model_dict(self, model_name: str) -> Dict[str, Any]: 103 | model = self.get_model(model_name) 104 | if model is None: 105 | raise ValueError(f"Model not found: {model_name}") 106 | 107 | return model.to_dict() 108 | 109 | def get_interleaved_model_names(self, tasks=None) -> List[str]: 110 | from itertools import zip_longest 111 | 112 | model_names = [] 113 | if tasks is None: 114 | model_names = [ 115 | self.t2i_model_names, 116 | self.t2v_model_names, 117 | self.i2i_model_names, 118 | self.i2v_model_names, 119 | ] 120 | else: 121 | for task in tasks: 122 | model_names.append(self.get_model_names_by_task(task)) 123 | 124 | shuffled_model_names = ( 125 | random.sample(names, len(names)) for names in model_names 126 | ) 127 | return [ 128 | m 129 | for quad in zip_longest(*shuffled_model_names) 130 | for m in quad 131 | if m is not None 132 | ] 133 | 134 | def get_modality(self, model_name: str) -> Modality: 135 | model = self.get_model(model_name) 136 | if model is None: 137 | raise ValueError(f"Model not found: {model_name}") 138 | return ( 139 | Modality.VIDEO 140 | if model.task in (ModelTask.TEXT_TO_VIDEO, ModelTask.IMAGE_TO_VIDEO) 141 | else Modality.IMAGE 142 | ) 143 | 144 | def get_task(self, model_name: str) -> str: 145 | model = self.get_model(model_name) 146 | if model is None: 147 | raise ValueError(f"Model not found: {model_name}") 148 | 149 | return model.task.value 150 | 151 | def get_output_media_type(self, model_name: str) -> MediaType: 152 | model = self.get_model(model_name) 153 | if model is None: 154 | raise ValueError(f"Model not found: {model_name}") 155 | 156 | return model.media_type 157 | -------------------------------------------------------------------------------- /gas/protocol/webhooks.py: -------------------------------------------------------------------------------- 1 | import time 2 | import threading 3 | from typing import Optional, Dict, Any, TYPE_CHECKING 4 | 5 | import requests 6 | import bittensor as bt 7 | 8 | from gas.protocol.epistula import generate_header 9 | 10 | if TYPE_CHECKING: 11 | from neurons.generator.task_manager import GenerationTask 12 | 13 | 14 | def send_success_webhook( 15 | task: "GenerationTask", 16 | result: Dict[str, Any], 17 | hotkey: bt.Keypair, 18 | external_ip: str, 19 | port: int, 20 | max_retries: int = 3, 21 | retry_delay: float = 2.0, 22 | timeout: float = 30.0 23 | ): 24 | binary_data = result.get("data") 25 | if not binary_data: 26 | bt.logging.error(f"send_success_webhook called with empty data for task {task.task_id}") 27 | return 28 | 29 | payload_size = len(binary_data) 30 | bt.logging.debug(f"Preparing webhook for task {task.task_id}: {payload_size} bytes") 31 | 32 | result_copy = { 33 | "data": binary_data, 34 | "metadata": result.get("metadata"), 35 | "url": result.get("url"), 36 | } 37 | 38 | def _send(): 39 | try: 40 | _send_webhook(task, result_copy, True, hotkey, external_ip, port, max_retries, retry_delay, timeout) 41 | except Exception as e: 42 | bt.logging.error(f"Failed to send success webhook for task {task.task_id}: {e}") 43 | 44 | threading.Thread(target=_send, daemon=True).start() 45 | 46 | 47 | def send_failure_webhook( 48 | task: "GenerationTask", 49 | hotkey: bt.Keypair, 50 | external_ip: str, 51 | port: int, 52 | max_retries: int = 3, 53 | retry_delay: float = 2.0, 54 | timeout: float = 30.0 55 | ): 56 | def _send(): 57 | try: 58 | _send_webhook(task, {}, False, hotkey, external_ip, port, max_retries, retry_delay, timeout) 59 | except Exception as e: 60 | bt.logging.error(f"Failed to send failure webhook for task {task.task_id}: {e}") 61 | 62 | threading.Thread(target=_send, daemon=True).start() 63 | 64 | 65 | def _send_webhook( 66 | task: "GenerationTask", 67 | result: Dict[str, Any], 68 | is_success: bool, 69 | hotkey: bt.Keypair, 70 | external_ip: str, 71 | port: int, 72 | max_retries: int, 73 | retry_delay: float, 74 | timeout: float 75 | ): 76 | for attempt in range(max_retries): 77 | try: 78 | success = _attempt_webhook_send(task, result, is_success, hotkey, timeout) 79 | if success: 80 | bt.logging.info(f"Webhook sent successfully for task {task.task_id}") 81 | return 82 | else: 83 | bt.logging.warning(f"Webhook attempt {attempt + 1} failed for task {task.task_id}") 84 | 85 | except Exception as e: 86 | bt.logging.error(f"Webhook attempt {attempt + 1} error for task {task.task_id}: {e}") 87 | 88 | if attempt < max_retries - 1: 89 | time.sleep(retry_delay ** attempt) 90 | 91 | bt.logging.error(f"All webhook attempts failed for task {task.task_id}") 92 | 93 | 94 | def _attempt_webhook_send( 95 | task: "GenerationTask", 96 | result: Dict[str, Any], 97 | is_success: bool, 98 | hotkey: bt.Keypair, 99 | timeout: float 100 | ) -> bool: 101 | 102 | binary_data = b"" 103 | try: 104 | if not is_success: 105 | content_type = "application/octet-stream" 106 | headers = { 107 | "Content-Type": content_type, 108 | "task-id": task.task_id, 109 | "task-status": "failed", 110 | "error-message": task.error_message or "Unknown error", 111 | **generate_header(hotkey, binary_data, task.signed_by), 112 | } 113 | else: 114 | binary_data = result.get("data") 115 | if not binary_data: 116 | bt.logging.error(f"Task {task.task_id} marked as successful but no binary data in result") 117 | return False 118 | 119 | payload_size = len(binary_data) 120 | bt.logging.info(f"Webhook payload for task {task.task_id}: {payload_size} bytes of {task.modality} data") 121 | 122 | if task.modality == "image": 123 | content_type = "image/png" 124 | elif task.modality == "video": 125 | content_type = "video/mp4" 126 | else: 127 | content_type = "application/octet-stream" 128 | 129 | headers = { 130 | "Content-Type": content_type, 131 | "Content-Length": str(payload_size), 132 | "task-id": task.task_id, 133 | "task-status": "completed", 134 | **generate_header(hotkey, binary_data, task.signed_by), 135 | } 136 | 137 | if result.get("metadata"): 138 | metadata = result["metadata"] 139 | for key, value in metadata.items(): 140 | header_key = f"x-meta-{key.replace('_', '-')}" 141 | headers[header_key] = str(value) 142 | 143 | bt.logging.info(f"Sending webhook to: {task.webhook_url}") 144 | response = requests.post( 145 | task.webhook_url, 146 | data=binary_data, 147 | headers=headers, 148 | timeout=timeout 149 | ) 150 | 151 | if response.status_code >= 300: 152 | bt.logging.warning( 153 | f"Webhook for task {task.task_id} returned status {response.status_code}: {response.text[:200]}" 154 | ) 155 | return False 156 | 157 | return True 158 | 159 | except requests.RequestException as e: 160 | bt.logging.warning(f"Webhook request failed for task {task.task_id}: {e}") 161 | return False 162 | -------------------------------------------------------------------------------- /neurons/generator/task_manager.py: -------------------------------------------------------------------------------- 1 | import time 2 | import uuid 3 | import threading 4 | from typing import Dict, List, Optional, Any 5 | from dataclasses import dataclass, asdict 6 | from enum import Enum 7 | 8 | 9 | class TaskStatus(Enum): 10 | PENDING = "pending" 11 | PROCESSING = "processing" 12 | COMPLETED = "completed" 13 | FAILED = "failed" 14 | 15 | 16 | @dataclass 17 | class GenerationTask: 18 | """Represents a generation task with all necessary information.""" 19 | task_id: str 20 | modality: str # "image", "video" 21 | status: TaskStatus 22 | prompt: str 23 | parameters: Dict[str, Any] 24 | webhook_url: str 25 | signed_by: str 26 | created_at: float 27 | 28 | # Optional fields 29 | input_data: Optional[bytes] = None 30 | result_data: Optional[bytes] = None 31 | result_url: Optional[str] = None 32 | error_message: Optional[str] = None 33 | started_at: Optional[float] = None 34 | completed_at: Optional[float] = None 35 | 36 | def to_dict(self) -> Dict[str, Any]: 37 | """Convert task to dictionary representation.""" 38 | data = asdict(self) 39 | data['status'] = self.status.value 40 | 41 | # Add processing time if available 42 | if self.started_at and self.completed_at: 43 | data['processing_time'] = self.completed_at - self.started_at 44 | elif self.started_at: 45 | data['processing_time'] = time.time() - self.started_at 46 | 47 | return data 48 | 49 | 50 | class TaskManager: 51 | """ 52 | Simple task manager for handling generation tasks. 53 | 54 | Features: 55 | - Thread-safe task operations 56 | - Automatic cleanup of old tasks 57 | - Status tracking through task lifecycle 58 | """ 59 | 60 | def __init__(self, max_task_age_hours: int = 24): 61 | self.tasks: Dict[str, GenerationTask] = {} 62 | self.max_task_age_hours = max_task_age_hours 63 | self._lock = threading.Lock() 64 | 65 | def create_task( 66 | self, 67 | modality: str, 68 | prompt: str, 69 | parameters: Dict[str, Any], 70 | webhook_url: str, 71 | signed_by: str, 72 | input_data: Optional[bytes] = None, 73 | ) -> str: 74 | """Create a new task and return its ID.""" 75 | task_id = str(uuid.uuid4()) 76 | 77 | task = GenerationTask( 78 | task_id=task_id, 79 | modality=modality, 80 | status=TaskStatus.PENDING, 81 | prompt=prompt, 82 | parameters=parameters, 83 | webhook_url=webhook_url, 84 | signed_by=signed_by, 85 | created_at=time.time(), 86 | input_data=input_data, 87 | ) 88 | 89 | with self._lock: 90 | self.tasks[task_id] = task 91 | 92 | return task_id 93 | 94 | def get_task(self, task_id: str) -> Optional[GenerationTask]: 95 | """Get a task by ID.""" 96 | with self._lock: 97 | return self.tasks.get(task_id) 98 | 99 | def get_pending_tasks(self) -> List[GenerationTask]: 100 | """Get all tasks with PENDING status.""" 101 | with self._lock: 102 | return [ 103 | task for task in self.tasks.values() 104 | if task.status == TaskStatus.PENDING 105 | ] 106 | 107 | def mark_task_processing(self, task_id: str) -> bool: 108 | """Mark a task as processing.""" 109 | with self._lock: 110 | if task_id in self.tasks: 111 | task = self.tasks[task_id] 112 | task.status = TaskStatus.PROCESSING 113 | task.started_at = time.time() 114 | return True 115 | return False 116 | 117 | def mark_task_completed( 118 | self, 119 | task_id: str, 120 | result_data: Optional[bytes] = None, 121 | result_url: Optional[str] = None 122 | ) -> bool: 123 | """Mark a task as completed with result data.""" 124 | with self._lock: 125 | if task_id in self.tasks: 126 | task = self.tasks[task_id] 127 | task.status = TaskStatus.COMPLETED 128 | task.completed_at = time.time() 129 | task.result_data = result_data 130 | task.result_url = result_url 131 | return True 132 | return False 133 | 134 | def mark_task_failed(self, task_id: str, error_message: str) -> bool: 135 | """Mark a task as failed with an error message.""" 136 | with self._lock: 137 | if task_id in self.tasks: 138 | task = self.tasks[task_id] 139 | task.status = TaskStatus.FAILED 140 | task.completed_at = time.time() 141 | task.error_message = error_message 142 | return True 143 | return False 144 | 145 | def cleanup_old_tasks(self) -> int: 146 | """Remove tasks older than max_task_age_hours. Returns number of tasks removed.""" 147 | cutoff_time = time.time() - (self.max_task_age_hours * 3600) 148 | 149 | with self._lock: 150 | old_task_ids = [ 151 | task_id for task_id, task in self.tasks.items() 152 | if task.created_at < cutoff_time 153 | ] 154 | 155 | for task_id in old_task_ids: 156 | del self.tasks[task_id] 157 | 158 | return len(old_task_ids) 159 | 160 | def get_task_stats(self) -> Dict[str, int]: 161 | """Get statistics about current tasks.""" 162 | with self._lock: 163 | stats = {"total": len(self.tasks)} 164 | 165 | for status in TaskStatus: 166 | stats[status.value] = sum( 167 | 1 for task in self.tasks.values() 168 | if task.status == status 169 | ) 170 | 171 | return stats 172 | -------------------------------------------------------------------------------- /gas/protocol/encoding.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | import ffmpeg 4 | import os 5 | import tempfile 6 | from typing import List 7 | from io import BytesIO 8 | from PIL import Image 9 | 10 | 11 | def image_to_bytes(img): 12 | """Convert image array to bytes using JPEG encoding with PIL. 13 | Args: 14 | img (np.ndarray): Image array of shape (C, H, W) or (H, W, C) 15 | Can be float32 [0,1] or uint8 [0,255] 16 | Returns: 17 | bytes: JPEG encoded image bytes 18 | str: Content type 'image/jpeg' 19 | """ 20 | # Convert float32 [0,1] to uint8 [0,255] if needed 21 | if img.dtype == np.float32: 22 | img = (img * 255).astype(np.uint8) 23 | elif img.dtype != np.uint8: 24 | raise ValueError(f"Image must be float32 or uint8, got {img.dtype}") 25 | 26 | if img.shape[0] == 3 and len(img.shape) == 3: # If in CHW format 27 | img = np.transpose(img, (1, 2, 0)) # CHW to HWC 28 | 29 | # Ensure we have a 3-channel image (H,W,3) 30 | if len(img.shape) == 2: 31 | # Convert grayscale to RGB 32 | img = np.stack([img, img, img], axis=2) 33 | elif img.shape[2] == 1: 34 | # Convert single channel to RGB 35 | img = np.concatenate([img, img, img], axis=2) 36 | elif img.shape[2] == 4: 37 | # Drop alpha channel 38 | img = img[:, :, :3] 39 | elif img.shape[2] != 3: 40 | raise ValueError(f"Expected 1, 3 or 4 channels, got {img.shape[2]}") 41 | 42 | pil_img = Image.fromarray(img) 43 | if pil_img.mode != "RGB": 44 | pil_img = pil_img.convert("RGB") 45 | 46 | buffer = BytesIO() 47 | pil_img.save(buffer, format="JPEG", quality=75) 48 | buffer.seek(0) 49 | 50 | return buffer.getvalue(), "image/jpeg" 51 | 52 | 53 | def video_to_bytes( 54 | video: np.ndarray, fps: int | None = None 55 | ) -> tuple[bytes, str]: 56 | """ 57 | Convert a (T, H, W, C) uint8/float32 video to MP4, but *first* pass each frame 58 | through Pillow JPEG → adds normal JPEG artefacts, then encodes losslessly. 59 | 60 | Returns: 61 | bytes: In‑memory MP4 file. 62 | str: MIME‑type ("video/mp4"). 63 | """ 64 | # ------------- 0. validation / normalisation ------------------------------- 65 | if video.dtype == np.float32: 66 | assert video.max() <= 1.0, video.max() 67 | video = (video * 255).clip(0, 255).astype(np.uint8) 68 | elif video.dtype != np.uint8: 69 | raise ValueError(f"Unsupported dtype: {video.dtype}") 70 | 71 | fps = fps or 30 72 | 73 | # TCHW → THWC 74 | if video.shape[1] <= 4 and video.shape[3] > 4: 75 | video = np.transpose(video, (0, 2, 3, 1)) 76 | 77 | if video.ndim != 4 or video.shape[3] not in (1, 3): 78 | raise ValueError(f"Expected shape (T, H, W, C), got {video.shape}") 79 | 80 | T, H, W, C = video.shape 81 | 82 | # ------------- 1. apply Pillow JPEG to every frame ------------------------- 83 | jpeg_degraded_frames: List[np.ndarray] = [] 84 | for idx, frame in enumerate(video): 85 | buf = BytesIO() 86 | Image.fromarray(frame).save( 87 | buf, 88 | format="JPEG", 89 | quality=75, 90 | subsampling=2, # 0=4:4:4, 1=4:2:2, 2=4:2:0 (Pillow default = 2) 91 | optimize=False, 92 | progressive=False, 93 | ) 94 | buf.seek(0) 95 | # decode back to RGB so FFmpeg sees the artefact‑laden pixels 96 | degraded = np.array(Image.open(buf).convert("RGB"), dtype=np.uint8) 97 | if degraded.shape != (H, W, 3): 98 | raise ValueError(f"Decoded shape mismatch at frame {idx}: {degraded.shape}") 99 | jpeg_degraded_frames.append(degraded) 100 | 101 | degraded_video = np.stack(jpeg_degraded_frames, axis=0) # (T,H,W,3) 102 | 103 | # ------------- 2. write raw RGB + encode losslessly ------------------------ 104 | with tempfile.TemporaryDirectory() as tmpdir: 105 | raw_path = os.path.join(tmpdir, "input.raw") 106 | video_path = os.path.join(tmpdir, "output.mp4") 107 | 108 | degraded_video.tofile(raw_path) # write as one big rawvideo blob 109 | 110 | try: 111 | ( 112 | ffmpeg.input( 113 | raw_path, 114 | format="rawvideo", 115 | pix_fmt="rgb24", 116 | s=f"{W}x{H}", 117 | r=fps, 118 | ) 119 | .output( 120 | video_path, 121 | vcodec="libx264rgb", 122 | crf=0, # mathematically lossless 123 | preset="veryfast", 124 | pix_fmt="rgb24", 125 | movflags="+faststart", 126 | ) 127 | .global_args("-y", "-hide_banner", "-loglevel", "error") 128 | .run() 129 | ) 130 | except ffmpeg.Error as e: 131 | raise RuntimeError( 132 | f"FFmpeg encoding failed:\n{e.stderr.decode(errors='ignore')}" 133 | ) from e 134 | 135 | with open(video_path, "rb") as f: 136 | video_bytes = f.read() 137 | 138 | return video_bytes, "video/mp4" 139 | 140 | 141 | def media_to_bytes(media, fps=30): 142 | """Convert image or video array to bytes, using PNG encoding for both. 143 | 144 | Args: 145 | media (np.ndarray): Either: 146 | - Image array of shape (C, H, W) 147 | - Video array of shape (T, C, H, W) 148 | Can be float32 [0,1] or uint8 [0,255] 149 | fps (int): Frames per second for video (default: 30) 150 | 151 | Returns: 152 | bytes: Encoded media bytes 153 | str: Content type (either 'image/png' or 'video/avi') 154 | """ 155 | if len(media.shape) == 3: # Image 156 | return image_to_bytes(media) 157 | elif len(media.shape) == 4: # Video 158 | return video_to_bytes(media, fps) 159 | else: 160 | raise ValueError( 161 | f"Invalid media shape: {media.shape}. Expected (C,H,W) for image or (T,C,H,W) for video." 162 | ) 163 | -------------------------------------------------------------------------------- /docs/ONNX.md: -------------------------------------------------------------------------------- 1 | # ONNX Model Creation Guide 2 | 3 | This guide explains how to create ONNX models for discriminative mining using the example scripts in `neurons/discriminator/onnx_examples`. 4 | 5 | ## Key Requirements 6 | 7 | **⚠️ IMPORTANT**: Your ONNX models must meet these requirements: 8 | 9 | 1. **Input Shape**: Specify fixed spatial dimensions for optimal batching 10 | - Image models: `['batch_size', 3, H, W]` where H and W are fixed (e.g., 224) 11 | - Video models: `['batch_size', 'frames', 3, H, W]` where H and W are fixed 12 | - ⚠️ If you use dynamic H/W axes, gasbench will default to 224x224 13 | 14 | 2. **Pixel Range**: Accept raw pixel values in range `[0-255]` 15 | - Gasbench handles preprocessing (shortest-edge resize, center crop, augmentations) 16 | - Your model wrapper should only normalize to [0, 1] and apply model-specific normalization 17 | 18 | 3. **Output Format**: Return logits for 3 classes `[real, synthetic, semisynthetic]` 19 | - Image models: `(batch_size, 3)` 20 | - Video models: `(batch_size, 3)` after temporal aggregation 21 | 22 | 23 | ## Example Scripts 24 | 25 | The `neurons/discriminator/onnx_examples` directory contains working examples: 26 | 27 | ### PyTorch Models 28 | - `pytorch_models/image_model.py` - Custom image classification model 29 | - `pytorch_models/video_model.py` - Custom video classification model 30 | 31 | ### HuggingFace Models 32 | - `huggingface_models/image_model.py` - Convert HuggingFace image models (e.g., ResNet50) 33 | - `huggingface_models/video_model.py` - Convert HuggingFace video models (e.g., VideoMAE) 34 | 35 | ## Quick Start 36 | 37 | 1. **Navigate to the examples directory:** 38 | ```bash 39 | cd neurons/discriminator/onnx_examples 40 | ``` 41 | 42 | 2. **Run an example script:** 43 | ```bash 44 | # For custom PyTorch models 45 | python pytorch_models/image_model.py 46 | python pytorch_models/video_model.py 47 | 48 | # For HuggingFace models 49 | python huggingface_models/image_model.py 50 | python huggingface_models/video_model.py 51 | ``` 52 | 53 | 3. **Check the output:** 54 | ```bash 55 | ls models/ 56 | # Should see: image_detector.onnx, video_detector.onnx 57 | ``` 58 | 59 | ## Preprocessing Pipeline 60 | 61 | Gasbench performs the following preprocessing before passing data to your model: 62 | 1. **Resize shortest edge** to target size (preserving aspect ratio) 63 | 2. **Center crop** to exact target size (H x W from your model spec) 64 | 3. **Random augmentations** (rotation, flip, crop, color jitter, etc.) 65 | 4. **Batching** with configurable batch size 66 | 67 | Your ONNX wrapper should only handle: 68 | - Normalization: `[0-255]` → `[0-1]` 69 | - Model-specific transforms (e.g., ImageNet mean/std) 70 | - **Video models**: Temporal aggregation 71 | 72 | ## Custom Models 73 | 74 | To create your own model: 75 | 76 | 1. **Inherit from `nn.Module`** and implement your architecture 77 | 2. **Wrap with preprocessing** (normalize, model-specific transforms, temporal aggregation) 78 | 3. **Export with fixed spatial dimensions** as shown in the examples 79 | 4. **Test with batched uint8 inputs** to ensure compatibility 80 | 81 | ## Example Export Code 82 | 83 | ### Image Model 84 | ```python 85 | # Create dummy input with raw pixel values (fixed spatial dims) 86 | dummy_input = torch.randint(0, 256, (1, 3, 224, 224), dtype=torch.uint8) 87 | 88 | # Export with fixed spatial dimensions for batching 89 | torch.onnx.export( 90 | wrapped_model, 91 | dummy_input, 92 | "models/image_detector.onnx", 93 | input_names=['input'], 94 | output_names=['logits'], 95 | dynamic_axes={ 96 | 'input': {0: 'batch_size'}, # Only batch_size is dynamic 97 | 'logits': {0: 'batch_size'} 98 | }, 99 | opset_version=11, 100 | do_constant_folding=True, 101 | export_params=True, 102 | keep_initializers_as_inputs=False 103 | ) 104 | ``` 105 | 106 | ### Video Model 107 | ```python 108 | # Create dummy input (B, T, C, H, W) with fixed spatial dims 109 | dummy_input = torch.randint(0, 256, (1, 8, 3, 224, 224), dtype=torch.uint8) 110 | 111 | # Export with dynamic batch and frames, fixed spatial dims 112 | torch.onnx.export( 113 | wrapped_model, 114 | dummy_input, 115 | "models/video_detector.onnx", 116 | input_names=['input'], 117 | output_names=['logits'], 118 | dynamic_axes={ 119 | 'input': {0: 'batch_size', 1: 'frames'}, # H, W are fixed 120 | 'logits': {0: 'batch_size'} 121 | }, 122 | opset_version=11, 123 | do_constant_folding=True, 124 | export_params=True, 125 | keep_initializers_as_inputs=False 126 | ) 127 | ``` 128 | 129 | ## Model Preprocessing Wrapper 130 | 131 | To perform input preprocessing or ouptut postprocessing, you can wrap your model: 132 | 133 | ```python 134 | class PreprocessingWrapper(nn.Module): 135 | def __init__(self, model, is_video=False): 136 | super().__init__() 137 | self.model = model 138 | 139 | def forward(self, x): 140 | # Input x: (B, T, C, H, W) for video, (B, C, H, W) for image 141 | # Values in range [0, 255] 142 | 143 | # Normalize to [0, 1] 144 | x = x.float() / 255.0 145 | 146 | # Apply model 147 | outputs = self.model(x) 148 | 149 | # Any necessary output postprocessing 150 | 151 | return outputs 152 | ``` 153 | 154 | The temporal aggregation prevents single-frame anomalies from dominating predictions. 155 | 156 | ## Packaging Your Models 157 | 158 | After creating your ONNX models, you need to package them into zip files before pushing to the network (keeps the system flexible in case we need to add supplemental files in the future): 159 | 160 | ```bash 161 | # Package image model 162 | zip image_detector.zip image_detector.onnx 163 | 164 | # Package video model 165 | zip video_detector.zip video_detector.onnx 166 | ``` 167 | 168 | Each zip file should contain only the corresponding ONNX model file. 169 | 170 | ## Next Steps 171 | 172 | Once you have your ONNX files packaged as zip files, follow the [Discriminative Mining Guide](Discriminative-Mining.md) to push them to the network. 173 | -------------------------------------------------------------------------------- /gas/protocol/epistula.py: -------------------------------------------------------------------------------- 1 | import json 2 | from hashlib import sha256 3 | from uuid import uuid4 4 | from math import ceil 5 | from typing import Annotated, Any, Dict, Optional 6 | 7 | import traceback 8 | import bittensor as bt 9 | import time 10 | import httpx 11 | from substrateinterface import Keypair 12 | from fastapi import Request, HTTPException 13 | 14 | 15 | EPISTULA_VERSION = str(2) 16 | MIN_VALIDATOR_STAKE = 20000 17 | 18 | 19 | def generate_header( 20 | hotkey: Keypair, 21 | body: Any, 22 | signed_for: Optional[str] = None, 23 | ) -> Dict[str, Any]: 24 | timestamp = round(time.time() * 1000) 25 | timestampInterval = ceil(timestamp / 1e4) * 1e4 26 | uuid = str(uuid4()) 27 | req_hash = None 28 | 29 | if isinstance(body, bytes): 30 | req_hash = sha256(body).hexdigest() 31 | elif isinstance(body, dict): 32 | body_copy = {} 33 | for key, value in body.items(): 34 | if isinstance(value, bytes): 35 | body_copy[key] = sha256(value).hexdigest() 36 | else: 37 | body_copy[key] = value 38 | req_hash = sha256(json.dumps(body_copy).encode("utf-8")).hexdigest() 39 | else: 40 | req_hash = sha256(json.dumps(body).encode("utf-8")).hexdigest() 41 | 42 | headers = { 43 | "Epistula-Version": EPISTULA_VERSION, 44 | "Epistula-Timestamp": str(timestamp), 45 | "Epistula-Uuid": uuid, 46 | "Epistula-Signed-By": hotkey.ss58_address, 47 | "Epistula-Request-Signature": "0x" 48 | + hotkey.sign(f"{req_hash}.{uuid}.{timestamp}.{signed_for or ''}").hex(), 49 | } 50 | if signed_for: 51 | headers["Epistula-Signed-For"] = signed_for 52 | headers["Epistula-Secret-Signature-0"] = ( 53 | "0x" + hotkey.sign(str(timestampInterval - 1) + "." + signed_for).hex() 54 | ) 55 | headers["Epistula-Secret-Signature-1"] = ( 56 | "0x" + hotkey.sign(str(timestampInterval) + "." + signed_for).hex() 57 | ) 58 | headers["Epistula-Secret-Signature-2"] = ( 59 | "0x" + hotkey.sign(str(timestampInterval + 1) + "." + signed_for).hex() 60 | ) 61 | return headers 62 | 63 | 64 | def verify_signature( 65 | signature, body: bytes, timestamp, uuid, signed_for, signed_by, now 66 | ) -> Optional[Annotated[str, "Error Message"]]: 67 | if not isinstance(signature, str): 68 | return "Invalid Signature" 69 | timestamp = int(timestamp) 70 | if not isinstance(timestamp, int): 71 | return "Invalid Timestamp" 72 | if not isinstance(signed_by, str): 73 | return "Invalid Sender key" 74 | if not isinstance(signed_for, str): 75 | return "Invalid receiver key" 76 | if not isinstance(uuid, str): 77 | return "Invalid uuid" 78 | if not isinstance(body, bytes): 79 | return "Body is not of type bytes" 80 | ALLOWED_DELTA_MS = 15000 81 | keypair = Keypair(ss58_address=signed_by) 82 | if timestamp + ALLOWED_DELTA_MS < now: 83 | staleness_ms = now - timestamp 84 | staleness_seconds = staleness_ms / 1000.0 85 | return f"Request is too stale: {staleness_seconds:.1f}s old (limit: {ALLOWED_DELTA_MS/1000.0}s)" 86 | message = f"{sha256(body).hexdigest()}.{uuid}.{timestamp}.{signed_for}" 87 | verified = keypair.verify(message, signature) 88 | if not verified: 89 | return "Signature Mismatch" 90 | return None 91 | 92 | 93 | def create_header_hook(hotkey, axon_hotkey, model): 94 | async def add_headers(request: httpx.Request): 95 | for key, header in generate_header(hotkey, request.read(), axon_hotkey).items(): 96 | request.headers[key] = header 97 | 98 | return add_headers 99 | 100 | 101 | async def _verify_request( 102 | request: Request, 103 | wallet: bt.Wallet, 104 | metagraph: bt.Metagraph, 105 | no_force_validator_permit: bool 106 | ): 107 | now = round(time.time() * 1000) 108 | 109 | signed_by = request.headers.get("Epistula-Signed-By") 110 | signed_for = request.headers.get("Epistula-Signed-For") 111 | client_ip = request.client.host if request.client else "unknown" 112 | 113 | if signed_for != wallet.hotkey.ss58_address: 114 | bt.logging.error(f"Request not intended for self from {signed_by} (IP: {client_ip})") 115 | raise HTTPException( 116 | status_code=400, detail="Bad Request, message is not intended for self" 117 | ) 118 | 119 | if signed_by not in metagraph.hotkeys: 120 | bt.logging.error(f"Signer not in metagraph: {signed_by} (IP: {client_ip})") 121 | raise HTTPException(status_code=401, detail="Signer not in metagraph") 122 | 123 | uid = metagraph.hotkeys.index(signed_by) 124 | stake = metagraph.S[uid].item() 125 | 126 | if not no_force_validator_permit and stake < MIN_VALIDATOR_STAKE: 127 | bt.logging.warning( 128 | f"Blacklisting request from {signed_by} [uid={uid}], not enough stake -- {stake}" 129 | ) 130 | raise HTTPException(status_code=401, detail=f"Stake below minimum: {stake}") 131 | 132 | body = await request.body() 133 | err = verify_signature( 134 | request.headers.get("Epistula-Request-Signature"), 135 | body, 136 | request.headers.get("Epistula-Timestamp"), 137 | request.headers.get("Epistula-Uuid"), 138 | signed_for, 139 | signed_by, 140 | now, 141 | ) 142 | 143 | if err: 144 | bt.logging.error(f"UID {uid} (IP: {client_ip}): {err}") 145 | raise HTTPException(status_code=400, detail=err) 146 | 147 | 148 | async def determine_epistula_version_and_verify( 149 | request: Request, 150 | wallet: bt.Wallet, 151 | metagraph: bt.Metagraph, 152 | no_force_validator_permit: bool 153 | ): 154 | version = request.headers.get("Epistula-Version") 155 | if version == EPISTULA_VERSION: 156 | await _verify_request(request, wallet, metagraph, no_force_validator_permit) 157 | return 158 | raise HTTPException(status_code=400, detail="Unknown Epistula version") 159 | 160 | 161 | def get_verifier( 162 | wallet: bt.Wallet, 163 | metagraph: bt.Metagraph, 164 | no_force_validator_permit: bool = False 165 | ): 166 | async def verifier(request: Request): 167 | await determine_epistula_version_and_verify( 168 | request, 169 | wallet, 170 | metagraph, 171 | no_force_validator_permit, 172 | ) 173 | return verifier 174 | -------------------------------------------------------------------------------- /validator.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const os = require('os'); 3 | 4 | // Helper functions 5 | function getPythonInterpreter() { 6 | const projectRoot = __dirname; 7 | const venvPython = path.join(projectRoot, '.venv', 'bin', 'python'); 8 | const fs = require('fs'); 9 | return fs.existsSync(venvPython) ? venvPython : 'python3'; 10 | } 11 | 12 | function getNetworkSettings(chainEndpoint) { 13 | if (chainEndpoint.includes('test')) return 379; 14 | if (chainEndpoint.includes('finney')) return 34; 15 | return null; 16 | } 17 | 18 | function getLogParam(loglevel) { 19 | switch (loglevel) { 20 | case 'trace': return '--logging.trace'; 21 | case 'debug': return '--logging.debug'; 22 | default: return '--logging.info'; 23 | } 24 | } 25 | 26 | function getAutoUpdateParam(autoUpdate) { 27 | return autoUpdate === 'true' ? '' : '--autoupdate-off'; 28 | } 29 | 30 | function getHeartbeatParam(heartbeat) { 31 | return heartbeat === 'true' ? '--heartbeat' : ''; 32 | } 33 | 34 | // Load environment variables 35 | require('dotenv').config({ path: path.resolve(__dirname, '.env.validator') }); 36 | 37 | // Check if old HUGGING_FACE_TOKEN is set and use it to set HUGGINGFACE_HUB_TOKEN 38 | if (process.env.HUGGING_FACE_TOKEN && !process.env.HUGGINGFACE_HUB_TOKEN) { 39 | process.env.HUGGINGFACE_HUB_TOKEN = process.env.HUGGING_FACE_TOKEN; 40 | } 41 | 42 | // Get configuration from environment with defaults 43 | const config = { 44 | // Wallet 45 | walletName: process.env.WALLET_NAME || 'default', 46 | walletHotkey: process.env.WALLET_HOTKEY || 'default', 47 | 48 | // Network 49 | chainEndpoint: process.env.CHAIN_ENDPOINT || '', 50 | callbackPort: process.env.CALLBACK_PORT || '10525', 51 | externalCallbackPort: process.env.EXTERNAL_CALLBACK_PORT || null, 52 | 53 | // Cache 54 | cacheDir: process.env.SN34_CACHE_DIR || path.join(os.homedir(), '.cache', 'sn34'), 55 | 56 | // Device 57 | device: process.env.DEVICE || 'cuda', 58 | 59 | // Logging 60 | loglevel: process.env.LOGLEVEL || 'info', 61 | 62 | // Features 63 | autoUpdate: process.env.AUTO_UPDATE || 'false', 64 | heartbeat: process.env.HEARTBEAT || 'false', 65 | 66 | // Service intervals 67 | scraperInterval: process.env.SCRAPER_INTERVAL || '300', 68 | datasetInterval: process.env.DATASET_INTERVAL || '1800', 69 | 70 | // API configuration 71 | benchmarkApiUrl: process.env.BENCHMARK_API_URL || 'https://gas.bitmind.ai', 72 | 73 | // Service selection 74 | startValidator: process.env.START_VALIDATOR !== 'false', 75 | startGenerator: process.env.START_GENERATOR !== 'false', 76 | startData: process.env.START_DATA !== 'false', 77 | }; 78 | 79 | // Determine netuid 80 | const netuid = getNetworkSettings(config.chainEndpoint); 81 | 82 | // Build command parameters 83 | const logParam = getLogParam(config.loglevel); 84 | const autoUpdateParam = getAutoUpdateParam(config.autoUpdate); 85 | const heartbeatParam = getHeartbeatParam(config.heartbeat); 86 | const pythonInterpreter = getPythonInterpreter(); 87 | 88 | // Project paths 89 | const projectRoot = __dirname; 90 | const validatorScript = path.join(projectRoot, 'neurons', 'validator', 'validator.py'); 91 | const generatorScript = path.join(projectRoot, 'neurons', 'validator', 'services', 'generator_service.py'); 92 | const dataScript = path.join(projectRoot, 'neurons', 'validator', 'services', 'data_service.py'); 93 | 94 | // Build apps array 95 | const apps = []; 96 | 97 | // Allow optional override of HF cache dir via env. Must be resolved before any Python starts. 98 | const HF_HOME_RESOLVED = process.env.HF_HOME 99 | || process.env.HUGGINGFACE_HOME 100 | || process.env.HUGGINGFACE_CACHE_DIR 101 | || path.join(os.homedir(), '.cache', 'huggingface'); 102 | 103 | // Common HF env 104 | const HF_ENV = { 105 | TRANSFORMERS_VERBOSITY: 'error', 106 | DIFFUSERS_VERBOSITY: 'error', 107 | TOKENIZERS_PARALLELISM: 'false', 108 | HF_HUB_VERBOSITY: 'error', 109 | ACCELERATE_LOG_LEVEL: 'error', 110 | HUGGINGFACE_HUB_TOKEN: process.env.HUGGINGFACE_HUB_TOKEN || process.env.HF_TOKEN, 111 | HF_HOME: HF_HOME_RESOLVED, 112 | HF_HUB_DISABLE_TELEMETRY: '1', 113 | }; 114 | 115 | // Validator service 116 | if (config.startValidator) { 117 | const validatorArgs = [ 118 | '--wallet.name', config.walletName, 119 | '--wallet.hotkey', config.walletHotkey, 120 | '--netuid', netuid.toString(), 121 | '--subtensor.chain_endpoint', config.chainEndpoint, 122 | '--neuron.callback_port', config.callbackPort, 123 | '--cache.base-dir', config.cacheDir, 124 | '--benchmark.api-url', config.benchmarkApiUrl, 125 | logParam, 126 | autoUpdateParam, 127 | ]; 128 | 129 | // Add external callback port if provided 130 | if (config.externalCallbackPort) { 131 | validatorArgs.push('--neuron.external-callback-port', config.externalCallbackPort); 132 | } 133 | 134 | if (heartbeatParam) { 135 | validatorArgs.push(heartbeatParam); 136 | } 137 | 138 | apps.push({ 139 | name: 'sn34-validator', 140 | script: validatorScript, 141 | interpreter: pythonInterpreter, 142 | args: validatorArgs.join(' '), 143 | env: { 144 | WANDB_API_KEY: process.env.WANDB_API_KEY, 145 | ...HF_ENV, 146 | }, 147 | watch: false, 148 | instances: 1, 149 | autorestart: true, 150 | }); 151 | } 152 | 153 | // Generator service 154 | if (config.startGenerator) { 155 | apps.push({ 156 | name: 'sn34-generator', 157 | script: generatorScript, 158 | interpreter: pythonInterpreter, 159 | args: [ 160 | '--wallet.name', config.walletName, 161 | '--wallet.hotkey', config.walletHotkey, 162 | '--cache.base-dir', config.cacheDir, 163 | '--device', config.device, 164 | '--log-level', config.loglevel, 165 | ].join(' '), 166 | env: { 167 | ...HF_ENV, 168 | }, 169 | watch: false, 170 | instances: 1, 171 | autorestart: true, 172 | }); 173 | } 174 | 175 | // Data service 176 | if (config.startData) { 177 | apps.push({ 178 | name: 'sn34-data', 179 | script: dataScript, 180 | interpreter: pythonInterpreter, 181 | args: [ 182 | '--cache.base-dir', config.cacheDir, 183 | '--chain-endpoint', config.chainEndpoint, 184 | '--scraper-interval', config.scraperInterval, 185 | '--dataset-interval', config.datasetInterval, 186 | '--log-level', config.loglevel, 187 | ].join(' '), 188 | env: { 189 | ...HF_ENV, 190 | TMPDIR: path.join(config.cacheDir, 'tmp'), 191 | TEMP: path.join(config.cacheDir, 'tmp'), 192 | TMP: path.join(config.cacheDir, 'tmp'), 193 | }, 194 | watch: false, 195 | instances: 1, 196 | autorestart: true, 197 | }); 198 | } 199 | 200 | module.exports = { 201 | apps, 202 | }; 203 | -------------------------------------------------------------------------------- /neurons/generator/services/service_registry.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import Dict, List, Optional, Any 3 | import bittensor as bt 4 | 5 | from .base_service import BaseGenerationService 6 | from .openai_service import OpenAIService 7 | from .openrouter_service import OpenRouterService 8 | from .stabilityai_service import StabilityAIService 9 | from .local_service import LocalService 10 | 11 | 12 | SERVICE_MAP = { 13 | "openai": OpenAIService, 14 | "openrouter": OpenRouterService, 15 | "local": LocalService, 16 | "stabilityai": StabilityAIService 17 | } 18 | 19 | 20 | class ServiceRegistry: 21 | """ 22 | Registry for managing generation services. 23 | 24 | Set per-modality service via env vars: 25 | IMAGE_SERVICE=openai|openrouter|local|none 26 | VIDEO_SERVICE=openai|openrouter|local|none 27 | 28 | Services: 29 | - openai: DALL-E 3 (requires OPENAI_API_KEY) 30 | - openrouter: Google Gemini via OpenRouter (requires OPEN_ROUTER_API_KEY) 31 | - local: Local Stable Diffusion models 32 | - none: Disable this modality (no service loaded) 33 | 34 | If not set, falls back to loading all available services. 35 | """ 36 | 37 | def __init__(self, config: Any = None): 38 | self.config = config 39 | self.services: Dict[str, BaseGenerationService] = {} # modality -> service 40 | self._all_services: List[BaseGenerationService] = [] # fallback list 41 | self._initialize_services() 42 | 43 | def _initialize_services(self): 44 | """Initialize services based on IMAGE_SERVICE and VIDEO_SERVICE env vars.""" 45 | image_service = os.getenv("IMAGE_SERVICE", "").lower().strip() 46 | video_service = os.getenv("VIDEO_SERVICE", "").lower().strip() 47 | 48 | if image_service or video_service: 49 | self._init_modality_services(image_service, video_service) 50 | else: 51 | self._init_all_services() 52 | 53 | def _init_modality_services(self, image_service: str, video_service: str): 54 | """Initialize specific services for each modality.""" 55 | initialized = set() 56 | 57 | if image_service == "none": 58 | bt.logging.info("IMAGE_SERVICE=none, no image service will be loaded") 59 | elif image_service: 60 | service = self._create_service(image_service, "image") 61 | if service: 62 | self.services["image"] = service 63 | initialized.add(image_service) 64 | 65 | if video_service == "none": 66 | bt.logging.info("VIDEO_SERVICE=none, no video service will be loaded") 67 | elif video_service: 68 | can_reuse = video_service in initialized and video_service != "local" 69 | if can_reuse: 70 | self.services["video"] = self.services.get("image") or self._create_service(video_service, "video") 71 | else: 72 | service = self._create_service(video_service, "video") 73 | if service: 74 | self.services["video"] = service 75 | 76 | def _create_service(self, service_name: str, modality: str) -> Optional[BaseGenerationService]: 77 | """Create and validate a service instance.""" 78 | if service_name not in SERVICE_MAP: 79 | bt.logging.error(f"Unknown service: {service_name}. Valid options: {list(SERVICE_MAP.keys())}") 80 | return None 81 | 82 | bt.logging.info(f"Initializing {service_name} for {modality}") 83 | service_class = SERVICE_MAP[service_name] 84 | 85 | try: 86 | if service_name == "local": 87 | service = service_class(self.config, target_modality=modality) 88 | else: 89 | service = service_class(self.config) 90 | if service.is_available(): 91 | bt.logging.success(f"✅ {service.name} ready for {modality}") 92 | return service 93 | else: 94 | bt.logging.error(f"❌ {service.name} configured for {modality} but not available (check API keys)") 95 | except Exception as e: 96 | bt.logging.error(f"Failed to initialize {service_name}: {e}") 97 | return None 98 | 99 | def _init_all_services(self): 100 | """Initialize all available services (fallback behavior).""" 101 | bt.logging.info("No IMAGE_SERVICE/VIDEO_SERVICE set, initializing all available services...") 102 | 103 | for name, service_class in SERVICE_MAP.items(): 104 | try: 105 | service = service_class(self.config) 106 | if service.is_available(): 107 | self._all_services.append(service) 108 | bt.logging.info(f"✅ {service.name} is available") 109 | else: 110 | bt.logging.info(f"❌ {service.name} is not available") 111 | except Exception as e: 112 | bt.logging.warning(f"Failed to initialize {name}: {e}") 113 | 114 | bt.logging.info(f"Initialized {len(self._all_services)} generation services") 115 | 116 | def get_service(self, modality: str) -> Optional[BaseGenerationService]: 117 | """Get the service for a modality.""" 118 | # Check explicit modality mapping first 119 | if modality in self.services: 120 | service = self.services[modality] 121 | bt.logging.debug(f"Using {service.name} for {modality}") 122 | return service 123 | 124 | # Fallback to scanning all services 125 | for service in self._all_services: 126 | if service.supports_modality(modality): 127 | bt.logging.debug(f"Using {service.name} for {modality}") 128 | return service 129 | 130 | bt.logging.warning(f"No service available for modality={modality}") 131 | return None 132 | 133 | def get_available_services(self) -> List[Dict[str, Any]]: 134 | """Get information about all available services.""" 135 | seen = set() 136 | result = [] 137 | 138 | for service in list(self.services.values()) + self._all_services: 139 | if service.name not in seen: 140 | seen.add(service.name) 141 | result.append(service.get_info()) 142 | return result 143 | 144 | def get_all_api_key_requirements(self) -> Dict[str, str]: 145 | """Get API key requirements from all services.""" 146 | all_requirements = { 147 | "IMAGE_SERVICE": "Service for images: openai, openrouter, local, or none", 148 | "VIDEO_SERVICE": "Service for videos: openai, openrouter, local, or none", 149 | } 150 | 151 | for name, service_class in SERVICE_MAP.items(): 152 | try: 153 | temp_service = service_class() 154 | all_requirements.update(temp_service.get_api_key_requirements()) 155 | except Exception as e: 156 | bt.logging.warning(f"Failed to get API key requirements from {name}: {e}") 157 | 158 | return all_requirements 159 | 160 | def reload_services(self): 161 | """Reload all services (useful for configuration changes).""" 162 | self.services.clear() 163 | self._all_services.clear() 164 | self._initialize_services() 165 | -------------------------------------------------------------------------------- /neurons/base.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from pathlib import Path 3 | from typing import Callable, List 4 | import bittensor as bt 5 | import copy 6 | import inspect 7 | import traceback 8 | import asyncio 9 | 10 | from bittensor.core.settings import SS58_FORMAT, TYPE_REGISTRY 11 | import signal 12 | 13 | from gas import ( 14 | __spec_version__, 15 | __version__, 16 | ) 17 | from gas.utils.metagraph import SubstrateConnectionManager 18 | from gas.types import NeuronType 19 | from gas.utils import ExitContext, on_block_interval 20 | from gas.config import ( 21 | add_args, 22 | add_miner_args, 23 | add_validator_args, 24 | validate_config_and_neuron_path, 25 | ) 26 | 27 | 28 | class BaseNeuron: 29 | """ 30 | Base neuron class with async substrate support and automatic reconnection. 31 | Provides clean async/await coordination throughout the application. 32 | """ 33 | config: "bt.config" 34 | neuron_type: NeuronType 35 | exit_context = ExitContext() 36 | next_sync_block = None 37 | block_callbacks: List[Callable] = [] 38 | substrate_manager: SubstrateConnectionManager = None 39 | substrate_task = None 40 | 41 | def check_registered(self): 42 | if not self.subtensor.is_hotkey_registered( 43 | netuid=self.config.netuid, 44 | hotkey_ss58=self.wallet.hotkey.ss58_address, 45 | ): 46 | bt.logging.error( 47 | f"Wallet: {self.wallet} is not registered on netuid {self.config.netuid}." 48 | f" Please register the hotkey using `btcli subnets register` before trying again" 49 | ) 50 | exit() 51 | 52 | @on_block_interval("epoch_length") 53 | async def maybe_sync_metagraph(self, block): 54 | self.check_registered() 55 | bt.logging.info("Resyncing Metagraph") 56 | self.metagraph.sync(subtensor=self.subtensor) 57 | 58 | async def run_callbacks(self, block): 59 | if ( 60 | hasattr(self, "initialization_complete") 61 | and not self.initialization_complete 62 | ): 63 | bt.logging.debug( 64 | f"Skipping callbacks at block {block} during initialization" 65 | ) 66 | return 67 | 68 | for callback in self.block_callbacks: 69 | try: 70 | res = callback(block) 71 | if inspect.isawaitable(res): 72 | await res 73 | except Exception as e: 74 | bt.logging.error( 75 | f"Failed running callback {callback.__name__}: {str(e)}" 76 | ) 77 | bt.logging.error(traceback.format_exc()) 78 | 79 | def __init__(self, config=None): 80 | bt.logging.info( 81 | f"Bittensor Version: {bt.__version__} | SN34 Version {__spec_version__}" 82 | ) 83 | 84 | parser = argparse.ArgumentParser() 85 | bt.wallet.add_args(parser) 86 | bt.subtensor.add_args(parser) 87 | bt.logging.add_args(parser) 88 | add_args(parser) 89 | 90 | if self.neuron_type == NeuronType.VALIDATOR: 91 | bt.axon.add_args(parser) 92 | add_validator_args(parser) 93 | if self.neuron_type == NeuronType.MINER: 94 | bt.axon.add_args(parser) 95 | add_miner_args(parser) 96 | 97 | self.config = bt.config(parser) 98 | if config: 99 | base_config = copy.deepcopy(config) 100 | self.config.merge(base_config) 101 | 102 | if hasattr(self.config, 'cache') and hasattr(self.config.cache, 'base_dir'): 103 | self.config.cache.base_dir = str(Path(self.config.cache.base_dir).expanduser()) 104 | 105 | validate_config_and_neuron_path(self.config) 106 | 107 | ## Add kill signals 108 | signal.signal(signal.SIGINT, self.exit_context.startExit) 109 | signal.signal(signal.SIGTERM, self.exit_context.startExit) 110 | 111 | ## LOGGING 112 | bt.logging(config=self.config, logging_dir=self.config.neuron.full_path) 113 | bt.logging.set_info() 114 | if self.config.logging.debug: 115 | bt.logging.set_debug(True) 116 | if self.config.logging.trace: 117 | bt.logging.set_trace(True) 118 | 119 | ## BITTENSOR INITIALIZATION 120 | bt.logging.success(self.config) 121 | self.wallet = bt.wallet(config=self.config) 122 | self.subtensor = bt.subtensor( 123 | config=self.config, network=self.config.subtensor.chain_endpoint 124 | ) 125 | self.metagraph = self.subtensor.metagraph(self.config.netuid) 126 | 127 | self.loop = asyncio.get_event_loop() 128 | bt.logging.debug(f"Wallet: {self.wallet}") 129 | bt.logging.debug(f"Subtensor: {self.subtensor}") 130 | bt.logging.debug(f"Metagraph: {self.metagraph}") 131 | 132 | ## CHECK IF REGG'D 133 | self.check_registered() 134 | self.uid = self.metagraph.hotkeys.index(self.wallet.hotkey.ss58_address) 135 | 136 | self.block_callbacks.append(self.maybe_sync_metagraph) 137 | self._init_substrate() 138 | 139 | def _init_substrate(self): 140 | """Initialize substrate connection manager - task will be started when event loop is running.""" 141 | self.substrate_manager = SubstrateConnectionManager( 142 | url=self.config.subtensor.chain_endpoint, 143 | ss58_format=SS58_FORMAT, 144 | type_registry=TYPE_REGISTRY 145 | ) 146 | self.substrate_task = None # Will be created when event loop is running 147 | bt.logging.info("Substrate connection manager initialized (task will start when event loop runs)") 148 | 149 | async def start_substrate_subscription(self): 150 | """Start the async substrate subscription - call this from async context.""" 151 | if self.substrate_task is None: 152 | self.substrate_task = self.substrate_manager.start_subscription_task(self.run_callbacks) 153 | bt.logging.info("🚀 Substrate subscription started") 154 | 155 | def check_substrate_connection(self): 156 | """Check substrate connection health and restart if needed.""" 157 | # Only check if task has been created (i.e., we're in async context) 158 | if self.substrate_task is not None and self.substrate_task.done(): 159 | bt.logging.info("Substrate connection lost, restarting...") 160 | try: 161 | self.substrate_task = self.substrate_manager.start_subscription_task(self.run_callbacks) 162 | bt.logging.info("Substrate connection restarted") 163 | except Exception as e: 164 | bt.logging.error(f"Failed to restart substrate task: {e}") 165 | raise 166 | 167 | async def shutdown_substrate(self): 168 | """Clean shutdown of substrate connection.""" 169 | bt.logging.info("Shutting down substrate connection...") 170 | 171 | if hasattr(self, 'substrate_manager') and self.substrate_manager: 172 | self.substrate_manager.stop() 173 | 174 | if hasattr(self, 'substrate_task') and self.substrate_task and not self.substrate_task.done(): 175 | self.substrate_task.cancel() 176 | try: 177 | await self.substrate_task 178 | except asyncio.CancelledError: 179 | pass 180 | 181 | bt.logging.info("Substrate shutdown complete") 182 | -------------------------------------------------------------------------------- /docs/Generative-Mining.md: -------------------------------------------------------------------------------- 1 | # Generative Mining Guide 2 | 3 | ## Before You Proceed 4 | 5 | Follow the [Installation Guide](Installation.md) to set up your environment before proceeding with generative mining operations. 6 | 7 | ## Generative Mining Overview 8 | 9 | Generative miners create synthetic media (images and videos) according to prompts from validators. Miners are rewarded based on their ability to: 10 | - Generate high-quality content that passes validation checks 11 | - Create convincing synthetic media that challenges discriminative miners 12 | - Respond quickly to generation requests 13 | - Maintain consistent uptime and availability 14 | 15 | Generative miners operate as FastAPI servers that receive generation requests from validators and respond asynchronously via webhooks. 16 | 17 | ## Configuration Setup 18 | 19 | ### Environment Configuration 20 | 21 | Create a `.env.gen_miner` file in the project root to configure your generative miner: 22 | 23 | ```bash 24 | # ======= Generative Miner Configuration ======= 25 | 26 | # Wallet Configuration (Required) 27 | BT_WALLET_NAME=your_wallet_name 28 | BT_WALLET_HOTKEY=your_hotkey_name 29 | 30 | # Network Configuration 31 | BT_CHAIN_ENDPOINT=wss://entrypoint-finney.opentensor.ai:443 32 | BT_NETUID=34 33 | 34 | # Axon Configuration 35 | BT_AXON_PORT=8093 36 | BT_AXON_IP=0.0.0.0 37 | BT_AXON_EXTERNAL_IP=auto 38 | 39 | # Miner Settings 40 | MINER_DEVICE=auto 41 | MINER_OUTPUT_DIR=/tmp/generated_content 42 | MINER_MAX_CONCURRENT_TASKS=5 43 | MINER_TASK_TIMEOUT=300 44 | 45 | # Logging 46 | BT_LOGGING_LEVEL=info 47 | 48 | # API Keys (Optional - for 3rd party services) 49 | # Configure API keys for external services, or use local generation 50 | OPENAI_API_KEY=your_openai_api_key 51 | OPEN_ROUTER_API_KEY=your_openrouter_api_key 52 | 53 | # Optional Settings 54 | AUTO_UPDATE=false 55 | MINER_NO_FORCE_VALIDATOR_PERMIT=false 56 | ``` 57 | 58 | ### Network Configuration 59 | 60 | **Mainnet (SN34)**: 61 | ```bash 62 | BT_CHAIN_ENDPOINT=wss://entrypoint-finney.opentensor.ai:443 63 | BT_NETUID=34 64 | ``` 65 | 66 | **Testnet (SN379)**: 67 | ```bash 68 | BT_CHAIN_ENDPOINT=wss://test.finney.opentensor.ai:443 69 | BT_NETUID=379 70 | ``` 71 | 72 | ## Generation Services 73 | 74 | Generative miners support multiple generation approaches. You can choose between external API services or local open source models. The default options are listed below, but miners are free to add any service they want. This is where generative miners can build an edge -- pick models that generate tough-to-classify examples to boost your fool-rates and your incentive. 75 | 76 | ### OpenAI Service (DALL-E) 77 | - **API Key**: `OPENAI_API_KEY` 78 | - **Supported**: Image generation 79 | - **Models**: DALL-E 3, DALL-E 2 80 | 81 | ### OpenRouter Service 82 | - **API Key**: `OPEN_ROUTER_API_KEY` 83 | - **Supported**: Image generation 84 | - **Models**: Google Gemini Flash Image Preview, various other models 85 | - **Website**: [OpenRouter.ai](https://openrouter.ai) 86 | 87 | ### Local Service (Open Source Models) 88 | - **API Key**: None required 89 | - **Supported**: Local model inference with open source models 90 | - **Models**: Stable Diffusion, FLUX, and other Hugging Face models 91 | - **Requirements**: GPU with sufficient VRAM (8GB+ recommended) 92 | - **Advantages**: No ongoing API costs, full control over models, privacy 93 | - **Note**: Requires additional model setup and local compute resources 94 | 95 | **Important**: Configure at least one generation method (API service OR local models) for your miner to be functional. Local generation with open source models is a cost-effective alternative to paid API services. 96 | 97 | ## Starting the Miner 98 | 99 | ### Using gascli (Recommended) 100 | 101 | First, activate the virtual environment: 102 | ```bash 103 | source .venv/bin/activate 104 | ``` 105 | 106 | Start the generative miner: 107 | ```bash 108 | # Start the miner 109 | gascli generator start 110 | 111 | # Using aliases 112 | gascli gen start 113 | gascli g start 114 | ``` 115 | 116 | ### Alternative Method 117 | 118 | You can also start the miner using PM2 directly: 119 | ```bash 120 | pm2 start gen_miner.config.js 121 | ``` 122 | 123 | ## Miner Management 124 | 125 | ### Status and Monitoring 126 | 127 | ```bash 128 | # Check miner status 129 | gascli generator status 130 | 131 | # View miner logs 132 | gascli generator logs 133 | 134 | # Follow logs in real-time 135 | gascli generator logs --follow 136 | 137 | # Show miner configuration and API key status 138 | gascli generator info 139 | ``` 140 | 141 | ### Starting and Stopping 142 | 143 | ```bash 144 | # Start the miner 145 | gascli generator start 146 | 147 | # Stop the miner 148 | gascli generator stop 149 | 150 | # Restart the miner 151 | gascli generator restart 152 | 153 | # Delete the miner process 154 | gascli generator delete 155 | ``` 156 | 157 | ## What Happens During Operation 158 | 159 | 1. **Service Registration**: The miner registers on the Bittensor network with your wallet and hotkey 160 | 2. **Service Initialization**: Available generation services are initialized (API services with key validation or local models) 161 | 3. **Request Handling**: Validators send generation requests to your miner's FastAPI endpoints 162 | 4. **Task Processing**: Requests are queued and processed using configured services (API or local models) 163 | 5. **Webhook Responses**: Results are sent back to validators via webhook URLs 164 | 6. **Reward Distribution**: Miners are scored based on generation quality and response time 165 | 166 | ### API Endpoints 167 | 168 | Your miner exposes these endpoints for validators: 169 | 170 | - `POST /gen_image` - Image generation requests 171 | - `POST /gen_video` - Video generation requests (coming soon) 172 | - `GET /health` - Health check endpoint 173 | - `GET /miner_info` - Miner information and capabilities 174 | - `GET /status/{task_id}` - Task status queries 175 | 176 | ## Configuration Parameters 177 | 178 | ### Core Settings 179 | 180 | - `BT_WALLET_NAME`: Your Bittensor wallet name 181 | - `BT_WALLET_HOTKEY`: Your wallet hotkey name 182 | - `BT_NETUID`: Subnet ID (34 for mainnet, 379 for testnet) 183 | - `BT_AXON_PORT`: Port for your miner's API server 184 | 185 | ### Performance Settings 186 | 187 | - `MINER_MAX_CONCURRENT_TASKS`: Maximum parallel generation tasks (default: 5) 188 | - `MINER_TASK_TIMEOUT`: Maximum time per task in seconds (default: 300) 189 | - `MINER_OUTPUT_DIR`: Directory for generated content and logs 190 | - `MINER_DEVICE`: Computing device (`auto`, `cuda`, `cpu`) 191 | 192 | ## Troubleshooting 193 | 194 | ### Common Issues 195 | 196 | **Miner fails to start**: 197 | - Check wallet configuration in `.env.gen_miner` 198 | - Ensure virtual environment is activated 199 | - Verify network connectivity to chain endpoint 200 | 201 | **No generation services available**: 202 | - Check API key configuration with `gascli generator info` 203 | - Verify API keys are valid and have sufficient credits 204 | - Consider using local open source models as an alternative to API services 205 | - Check logs for service initialization errors 206 | 207 | **Low rewards**: 208 | - Monitor generation quality and response times 209 | - Ensure stable internet connection and uptime 210 | - Consider upgrading API service plans for better performance 211 | 212 | ### Getting Help 213 | 214 | ```bash 215 | # General help 216 | gascli generator --help 217 | 218 | # Command-specific help 219 | gascli generator start --help 220 | gascli generator logs --help 221 | 222 | # Check service status 223 | gascli generator info 224 | ``` 225 | 226 | **Note**: Remember to activate the virtual environment with `source .venv/bin/activate` before running any `gascli` commands. 227 | -------------------------------------------------------------------------------- /neurons/generator/services/stabilityai_service.py: -------------------------------------------------------------------------------- 1 | import os 2 | import io 3 | import time 4 | import requests 5 | import bittensor as bt 6 | from typing import Dict, Any, Optional 7 | 8 | from PIL import Image 9 | import c2pa 10 | 11 | from .base_service import BaseGenerationService 12 | from ..task_manager import GenerationTask 13 | 14 | 15 | class Models: 16 | """Symbolic constants for StabilityAI models.""" 17 | 18 | SD35_MEDIUM = "sd3.5-medium" 19 | SD35_LARGE = "sd3.5-large" 20 | ULTRA = "ultra" 21 | CORE = "core" 22 | 23 | 24 | MODEL_INFO = { 25 | Models.SD35_MEDIUM: { 26 | "endpoint": "https://api.stability.ai/v2beta/stable-image/generate/sd3", 27 | "name": "Stable Diffusion 3.5 Medium", 28 | "family": "sd3.5", 29 | "description": "Balanced high-quality diffusion model", 30 | }, 31 | Models.SD35_LARGE: { 32 | "endpoint": "https://api.stability.ai/v2beta/stable-image/generate/sd3", 33 | "name": "Stable Diffusion 3.5 Large", 34 | "family": "sd3.5", 35 | "description": "Higher-quality large model", 36 | }, 37 | Models.ULTRA: { 38 | "endpoint": "https://api.stability.ai/v2beta/stable-image/generate/ultra", 39 | "name": "Stability Ultra", 40 | "family": "ultra", 41 | "description": "Highest-quality proprietary model", 42 | }, 43 | Models.CORE: { 44 | "endpoint": "https://api.stability.ai/v2beta/stable-image/generate/core", 45 | "name": "Stability Core", 46 | "family": "core", 47 | "description": "Fast, efficient core model", 48 | }, 49 | } 50 | 51 | class StabilityAIService(BaseGenerationService): 52 | """ 53 | Stability AI generation service for Ultra, Core, and SD 3.5 family models. 54 | 55 | Features: 56 | - Binary image generation 57 | - Embedded C2PA manifest extraction 58 | - Full MIME-type automatic detection 59 | - Model → endpoint mapping 60 | """ 61 | 62 | # Allowed output formats 63 | VALID_FORMATS = {"png", "jpeg", "webp"} 64 | 65 | def __init__(self, config: Any = None): 66 | super().__init__(config) 67 | 68 | self.api_key = os.getenv("STABILITY_API_KEY") 69 | self.timeout = 90 70 | self.default_model = Models.SD35_MEDIUM 71 | 72 | if not self.api_key: 73 | bt.logging.warning("STABILITY_API_KEY not found.") 74 | else: 75 | bt.logging.info("StabilityAIService initialized with API key") 76 | 77 | # --------------------------------------------------------------------- 78 | # Base methods 79 | # --------------------------------------------------------------------- 80 | def is_available(self) -> bool: 81 | return self.api_key is not None and self.api_key.strip() != "" 82 | 83 | def supports_modality(self, modality: str) -> bool: 84 | return modality == "image" 85 | 86 | def get_supported_tasks(self) -> Dict[str, list]: 87 | return { 88 | "image": ["image_generation"], 89 | "video": [] # StabilityAI doesn't support video generation yet 90 | } 91 | 92 | def get_api_key_requirements(self) -> Dict[str, str]: 93 | return {"STABILITY_API_KEY": "API key for StabilityAI image generation"} 94 | 95 | # --------------------------------------------------------------------- 96 | # Processing logic 97 | # --------------------------------------------------------------------- 98 | def process(self, task: GenerationTask) -> Dict[str, Any]: 99 | if task.modality != "image": 100 | raise ValueError(f"StabilityAIService does not support modality: {task.modality}") 101 | 102 | return self._generate_image(task) 103 | 104 | # --------------------------------------------------------------------- 105 | # C2PA extractor 106 | # --------------------------------------------------------------------- 107 | def _extract_c2pa_metadata(self, img_bytes: bytes, output_format: str) -> Optional[Dict[str, Any]]: 108 | """Extract embedded C2PA manifest from image bytes.""" 109 | 110 | mime_map = { 111 | "png": "image/png", 112 | "jpeg": "image/jpeg", 113 | "jpg": "image/jpeg", 114 | "webp": "image/webp", 115 | } 116 | 117 | mime_type = mime_map.get(output_format.lower(), "application/octet-stream") 118 | 119 | try: 120 | with io.BytesIO(img_bytes) as f: 121 | with c2pa.Reader(mime_type, f) as reader: 122 | return reader.json() 123 | except Exception as e: 124 | bt.logging.warning(f"No C2PA metadata detected or failed to read: {e}") 125 | return None 126 | 127 | def _get_endpoint(self, model: str) -> str: 128 | if model not in MODEL_INFO: 129 | raise ValueError(f"Unknown StabilityAI model: {model}") 130 | return MODEL_INFO[model]["endpoint"] 131 | 132 | # --------------------------------------------------------------------- 133 | # Image generation core 134 | # --------------------------------------------------------------------- 135 | def _generate_image(self, task: GenerationTask) -> Dict[str, Any]: 136 | try: 137 | params = task.parameters or {} 138 | 139 | model = params.get("model", self.default_model) 140 | prompt = task.prompt 141 | 142 | bt.logging.info(f"StabilityAI generating image with model={model}") 143 | 144 | if model not in MODEL_INFO: 145 | raise ValueError(f"Unknown StabilityAI model: {model}. " 146 | f"Available models: {list(MODEL_INFO.keys())}") 147 | 148 | url = self._get_endpoint(model) 149 | 150 | output_format = params.get("format", "png") 151 | if output_format not in self.VALID_FORMATS: 152 | output_format = "png" 153 | 154 | api_data = { 155 | "prompt": prompt, 156 | "output_format": output_format, 157 | "model": model, 158 | } 159 | 160 | # Optional parameters 161 | if "negative_prompt" in params: 162 | api_data["negative_prompt"] = params["negative_prompt"] 163 | if "aspect_ratio" in params: 164 | api_data["aspect_ratio"] = params["aspect_ratio"] 165 | if "seed" in params: 166 | api_data["seed"] = str(params["seed"]) 167 | 168 | headers = { 169 | "authorization": f"Bearer {self.api_key}", 170 | "accept": "image/*", 171 | } 172 | 173 | # Multipart requirement 174 | files = {"none": ""} 175 | 176 | start_time = time.time() 177 | response = requests.post( 178 | url, 179 | headers=headers, 180 | data=api_data, 181 | files=files, 182 | timeout=self.timeout, 183 | ) 184 | 185 | if response.status_code != 200: 186 | raise RuntimeError(f"Stability API error {response.status_code}: {response.text}") 187 | 188 | img_bytes = response.content 189 | gen_time = time.time() - start_time 190 | 191 | # Extract C2PA (embedded in image) 192 | c2pa_metadata = self._extract_c2pa_metadata(img_bytes, output_format) 193 | 194 | # Return final miner-compatible result 195 | return { 196 | "data": img_bytes, 197 | "metadata": { 198 | "model": model, 199 | "provider": "stability.ai", 200 | "format": output_format.upper(), 201 | "generation_time": gen_time, 202 | "c2pa": c2pa_metadata, 203 | } 204 | } 205 | 206 | except Exception as e: 207 | bt.logging.error(f"StabilityAI image generation failed: {e}") 208 | raise 209 | 210 | def get_service_info(self) -> Dict[str, Any]: 211 | """Return information about this service.""" 212 | return { 213 | "name": "StabilityAI", 214 | "type": "api", 215 | "provider": "api.stability.ai", 216 | "available": self.is_available(), 217 | "supported_tasks": self.get_supported_tasks(), 218 | "default_model": self.default_model 219 | } 220 | -------------------------------------------------------------------------------- /gas/protocol/model_uploads.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import json 3 | from pathlib import Path 4 | from typing import Optional 5 | 6 | import bittensor as bt 7 | import requests 8 | 9 | from gas.protocol.epistula import generate_header 10 | 11 | 12 | def calculate_sha256(data: bytes) -> str: 13 | return hashlib.sha256(data).hexdigest() 14 | 15 | 16 | def generate_presigned_url( 17 | wallet: bt.wallet, 18 | upload_endpoint: str, 19 | filename: str, 20 | file_size: int, 21 | file_hash: str, 22 | content_type: Optional[str] = None, 23 | modality: Optional[str] = None 24 | ) -> dict: 25 | """Generate presigned upload URL from the API with optional modality parameter.""" 26 | 27 | # Prepare request payload 28 | payload = { 29 | 'filename': filename, 30 | 'file_size': file_size, 31 | 'expected_hash': file_hash, 32 | } 33 | if content_type: 34 | payload['content_type'] = content_type 35 | if modality: 36 | payload['modality'] = modality 37 | 38 | payload_json = json.dumps(payload, separators=(',', ':')) 39 | payload_bytes = payload_json.encode('utf-8') 40 | 41 | headers = generate_header(wallet.hotkey, payload_bytes) 42 | headers['Content-Type'] = 'application/json' 43 | 44 | print(f"\n📡 Requesting presigned URL...") 45 | print(f" Request signed by: {headers['Epistula-Signed-By']}") 46 | 47 | # Make the presigned URL request 48 | try: 49 | presigned_endpoint = upload_endpoint.rstrip('/') + '/presigned' 50 | print(f" Endpoint: {presigned_endpoint}") 51 | 52 | response = requests.post( 53 | presigned_endpoint, 54 | data=payload_bytes, 55 | headers=headers, 56 | timeout=30 57 | ) 58 | 59 | # Parse response 60 | try: 61 | result = response.json() 62 | except json.JSONDecodeError: 63 | result = {"error": "Invalid JSON response", "text": response.text} 64 | 65 | return { 66 | "status_code": response.status_code, 67 | "success": response.status_code == 200, 68 | "response": result 69 | } 70 | 71 | except requests.exceptions.RequestException as e: 72 | return { 73 | "status_code": 0, 74 | "success": False, 75 | "response": {"error": f"Request failed: {str(e)}"} 76 | } 77 | 78 | 79 | def upload_to_r2(presigned_url: str, file_content: bytes, content_type: str = 'application/octet-stream') -> dict: 80 | """Upload file directly to R2 using presigned URL.""" 81 | 82 | print(f"🚢 Uploading file to R2...") 83 | print(f" Content type: {content_type}") 84 | print(f" File size: {len(file_content)} bytes") 85 | 86 | try: 87 | # Upload using PUT request to presigned URL 88 | response = requests.put( 89 | presigned_url, 90 | data=file_content, 91 | headers={'Content-Type': content_type}, 92 | timeout=300 # 5 minutes for large files 93 | ) 94 | 95 | return { 96 | "status_code": response.status_code, 97 | "success": response.status_code == 200, 98 | "response": { 99 | "message": "Upload successful" if response.status_code == 200 else "Upload failed", 100 | "etag": response.headers.get('ETag', ''), 101 | "response_text": response.text 102 | } 103 | } 104 | 105 | except requests.exceptions.RequestException as e: 106 | return { 107 | "status_code": 0, 108 | "success": False, 109 | "response": {"error": f"Upload failed: {str(e)}"} 110 | } 111 | 112 | 113 | def confirm_upload(wallet: bt.wallet, upload_endpoint: str, model_id: int, file_hash: str) -> dict: 114 | """Confirm file upload and finalize model record.""" 115 | 116 | payload = { 117 | 'model_id': model_id, 118 | 'file_hash': file_hash 119 | } 120 | 121 | payload_json = json.dumps(payload, separators=(',', ':')) 122 | payload_bytes = payload_json.encode('utf-8') 123 | 124 | headers = generate_header(wallet.hotkey, payload_bytes) 125 | headers['Content-Type'] = 'application/json' 126 | 127 | print(f"📡 Confirming upload...") 128 | print(f" Model ID: {model_id}") 129 | print(f" File hash: {file_hash}") 130 | 131 | try: 132 | confirm_endpoint = upload_endpoint.rstrip('/') + '/confirm' 133 | response = requests.post( 134 | confirm_endpoint, 135 | data=payload_bytes, 136 | headers=headers, 137 | timeout=30 138 | ) 139 | 140 | try: 141 | result = response.json() 142 | except json.JSONDecodeError: 143 | result = {"error": "Invalid JSON response", "text": response.text} 144 | 145 | return { 146 | "status_code": response.status_code, 147 | "success": response.status_code == 200, 148 | "response": result 149 | } 150 | 151 | except requests.exceptions.RequestException as e: 152 | return { 153 | "status_code": 0, 154 | "success": False, 155 | "response": {"error": f"Request failed: {str(e)}"} 156 | } 157 | 158 | 159 | def upload_single_modality( 160 | wallet: bt.wallet, 161 | file_path: str, 162 | modality: str, 163 | upload_endpoint: str 164 | ) -> dict: 165 | """Upload a single modality file (image or video model).""" 166 | file_path_obj = Path(file_path) 167 | if not file_path_obj.exists(): 168 | raise FileNotFoundError(f"File not found: {file_path}") 169 | 170 | with open(file_path_obj, 'rb') as f: 171 | file_content = f.read() 172 | 173 | file_hash = calculate_sha256(file_content) 174 | file_size = len(file_content) 175 | filename = file_path_obj.name 176 | 177 | print(f"\n{'='*70}") 178 | print(f"UPLOADING {modality.upper()} MODEL") 179 | print(f"{'='*70}") 180 | print(f"📁 File: {filename}") 181 | print(f"📊 Size: {file_size} bytes ({file_size / 1024 / 1024:.2f} MB)") 182 | print(f"🔐 Hash: {file_hash}") 183 | 184 | # Step 1: Generate presigned URL with modality 185 | print(f"\n[1/3] Generating presigned URL...") 186 | presigned_result = generate_presigned_url( 187 | wallet, 188 | upload_endpoint, 189 | filename, 190 | file_size, 191 | file_hash, 192 | 'application/octet-stream', 193 | modality 194 | ) 195 | 196 | if not presigned_result['success']: 197 | return { 198 | "success": False, 199 | "modality": modality, 200 | "step": "presigned_url_generation", 201 | "error": presigned_result['response'].get('error', 'Unknown error'), 202 | "response": presigned_result['response'] 203 | } 204 | 205 | presigned_data = presigned_result['response']['data'] 206 | model_id = presigned_data['model_id'] 207 | presigned_url = presigned_data['presigned_url'] 208 | r2_key = presigned_data['r2_key'] 209 | 210 | print(f"✅ Presigned URL generated!") 211 | print(f" Model ID: {model_id}") 212 | print(f" R2 Key: {r2_key}") 213 | print(f"\n[2/3] Uploading to R2...") 214 | upload_result = upload_to_r2(presigned_url, file_content, 'application/octet-stream') 215 | 216 | if not upload_result['success']: 217 | return { 218 | "success": False, 219 | "modality": modality, 220 | "step": "r2_upload", 221 | "model_id": model_id, 222 | "error": upload_result['response'].get('error', 'Unknown error'), 223 | "response": upload_result['response'] 224 | } 225 | 226 | print(f"✅ File uploaded to R2!") 227 | print(f" ETag: {upload_result['response'].get('etag', 'N/A')}") 228 | 229 | print(f"\n[3/3] Confirming upload...") 230 | confirm_result = confirm_upload(wallet, upload_endpoint, model_id, file_hash) 231 | 232 | if not confirm_result['success']: 233 | return { 234 | "success": False, 235 | "modality": modality, 236 | "step": "upload_confirmation", 237 | "model_id": model_id, 238 | "error": confirm_result['response'].get('error', 'Unknown error'), 239 | "response": confirm_result['response'] 240 | } 241 | 242 | print(f"✅ Upload confirmed!") 243 | return { 244 | "success": True, 245 | "modality": modality, 246 | "model_id": model_id, 247 | "r2_key": r2_key, 248 | "file_hash": file_hash, 249 | "file_size": file_size 250 | } -------------------------------------------------------------------------------- /docs/Incentive.md: -------------------------------------------------------------------------------- 1 | # Incentive Mechanism 2 | 3 | ## Benchmark Runs 4 | Submitted discriminator miners are evaluated against a subset of the data sources listed below. A portion of the evaluation data comes from generative miners, who are rewarded based on their ability to submit data that both pass validator sanity checks (prompt alignment, etc.) and fool discriminators in benchmark runs. 5 | 6 |
7 | Evaluation Datasets 8 | 9 | ### Image Datasets 10 | 11 | **Real Images:** 12 | - [drawthingsai/megalith-10m](https://huggingface.co/datasets/drawthingsai/megalith-10m) 13 | - [bitmind/bm-eidon-image](https://huggingface.co/datasets/bitmind/bm-eidon-image) 14 | - [bitmind/bm-real](https://huggingface.co/datasets/bitmind/bm-real) 15 | - [bitmind/open-image-v7-256](https://huggingface.co/datasets/bitmind/open-image-v7-256) 16 | - [bitmind/celeb-a-hq](https://huggingface.co/datasets/bitmind/celeb-a-hq) 17 | - [bitmind/ffhq-256](https://huggingface.co/datasets/bitmind/ffhq-256) 18 | - [bitmind/MS-COCO-unique-256](https://huggingface.co/datasets/bitmind/MS-COCO-unique-256) 19 | - [bitmind/AFHQ](https://huggingface.co/datasets/bitmind/AFHQ) 20 | - [bitmind/lfw](https://huggingface.co/datasets/bitmind/lfw) 21 | - [bitmind/caltech-256](https://huggingface.co/datasets/bitmind/caltech-256) 22 | - [bitmind/caltech-101](https://huggingface.co/datasets/bitmind/caltech-101) 23 | - [bitmind/dtd](https://huggingface.co/datasets/bitmind/dtd) 24 | - [bitmind/idoc-mugshots-images](https://huggingface.co/datasets/bitmind/idoc-mugshots-images) 25 | 26 | **Synthetic Images:** 27 | - [bitmind/JourneyDB](https://huggingface.co/datasets/bitmind/JourneyDB) 28 | - [bitmind/GenImage_MidJourney](https://huggingface.co/datasets/bitmind/GenImage_MidJourney) 29 | - [bitmind/bm-aura-imagegen](https://huggingface.co/datasets/bitmind/bm-aura-imagegen) 30 | - [bitmind/bm-imagine](https://huggingface.co/datasets/bitmind/bm-imagine) 31 | - [Yejy53/Echo-4o-Image](https://huggingface.co/datasets/Yejy53/Echo-4o-Image) 32 | 33 | **Semi-synthetic Images:** 34 | - [bitmind/face-swap](https://huggingface.co/datasets/bitmind/face-swap) 35 | 36 | ### Video Datasets 37 | 38 | **Real Videos:** 39 | - [bitmind/bm-eidon-video](https://huggingface.co/datasets/bitmind/bm-eidon-video) 40 | - [shangxd/imagenet-vidvrd](https://huggingface.co/datasets/shangxd/imagenet-vidvrd) 41 | - [nkp37/OpenVid-1M](https://huggingface.co/datasets/nkp37/OpenVid-1M) 42 | - [facebook/PE-Video](https://huggingface.co/datasets/facebook/PE-Video) 43 | 44 | **Semi-synthetic Videos:** 45 | - [bitmind/semisynthetic-video](https://huggingface.co/datasets/bitmind/semisynthetic-video) 46 | 47 | **Synthetic Videos:** 48 | - [Rapidata/text-2-video-human-preferences-veo3](https://huggingface.co/datasets/Rapidata/text-2-video-human-preferences-veo3) 49 | - [Rapidata/text-2-video-human-preferences-veo2](https://huggingface.co/datasets/Rapidata/text-2-video-human-preferences-veo2) 50 | - [bitmind/aura-video](https://huggingface.co/datasets/bitmind/aura-video) 51 | - [bitmind/aislop-videos](https://huggingface.co/datasets/bitmind/aislop-videos) 52 | 53 |
54 | 55 |
56 | Generative Models 57 | 58 | The following models run by validators to produce a continual, fresh stream of synthetic and semisynthetic data. The outputs of these models are uploaded at regular intervals to public datasets in the [GAS-Station](https://huggingface.co/gasstation) Hugging Face org for miner training and evaluation. 59 | 60 | ### Text-to-Image Models 61 | 62 | - [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) 63 | - [SG161222/RealVisXL_V4.0](https://huggingface.co/SG161222/RealVisXL_V4.0) 64 | - [Corcelio/mobius](https://huggingface.co/Corcelio/mobius) 65 | - [prompthero/openjourney-v4](https://huggingface.co/prompthero/openjourney-v4) 66 | - [cagliostrolab/animagine-xl-3.1](https://huggingface.co/cagliostrolab/animagine-xl-3.1) 67 | - [runwayml/stable-diffusion-v1-5](https://huggingface.co/runwayml/stable-diffusion-v1-5) + [Kvikontent/midjourney-v6](https://huggingface.co/Kvikontent/midjourney-v6) LoRA 68 | - [black-forest-labs/FLUX.1-dev](https://huggingface.co/black-forest-labs/FLUX.1-dev) 69 | - [DeepFloyd/IF](https://huggingface.co/DeepFloyd/IF) 70 | - [deepseek-ai/Janus-Pro-7B](https://huggingface.co/deepseek-ai/Janus-Pro-7B) 71 | - [THUDM/CogView4-6B](https://huggingface.co/THUDM/CogView4-6B) 72 | 73 | ### Image-to-Image Models 74 | 75 | - [diffusers/stable-diffusion-xl-1.0-inpainting-0.1](https://huggingface.co/diffusers/stable-diffusion-xl-1.0-inpainting-0.1) 76 | - [Lykon/dreamshaper-8-inpainting](https://huggingface.co/Lykon/dreamshaper-8-inpainting) 77 | 78 | ### Text-to-Video Models 79 | 80 | - [tencent/HunyuanVideo](https://huggingface.co/tencent/HunyuanVideo) 81 | - [genmo/mochi-1-preview](https://huggingface.co/genmo/mochi-1-preview) 82 | - [THUDM/CogVideoX-5b](https://huggingface.co/THUDM/CogVideoX-5b) 83 | - [ByteDance/AnimateDiff-Lightning](https://huggingface.co/ByteDance/AnimateDiff-Lightning) 84 | - [Wan-AI/Wan2.2-TI2V-5B-Diffusers](https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B-Diffusers) 85 | 86 | ### Image-to-Video Models 87 | 88 | - [THUDM/CogVideoX1.5-5B-I2V](https://huggingface.co/THUDM/CogVideoX1.5-5B-I2V) 89 | 90 |
91 | 92 | 93 | ## Generator Rewards 94 | 95 | The generator incentive mechanism combines two components: a base reward for passing data validation checks, and a multiplier based on adversarial performance against discriminators. 96 | 97 | ### Base Reward (Data Validation) 98 | 99 | Generators receive a base reward based on their data verification pass rate: 100 | 101 | $$R_{\text{base}} = p \cdot \min(n, 10)$$ 102 | 103 | Where: 104 | - $p$ = pass rate (proportion of generated content that passes validation) 105 | - $n$ = number of verified samples (`min(p, 10)` creates a rampup of incentive for the first 10 samples) 106 | 107 | ### Fool Rate Multiplier (Adversarial Performance) 108 | 109 | Generators earn additional rewards by successfully fooling discriminators. The multiplier is calculated as: 110 | 111 | $$M = \max(0, \min(2.0, f \cdot s))$$ 112 | 113 | Where: 114 | - $f$ = fool rate = $\frac{N_{\text{fooled}}}{N_{\text{fooled}} + N_{\text{not fooled}}}$ 115 | - $s$ = sample size multiplier 116 | 117 | The sample size multiplier encourages generators to be evaluated on more samples, similar to the sample size ramp used in the base reward. 118 | 119 | $$s = \begin{cases} 120 | \max(0.5, \frac{c}{20}) & \text{if } c < 20 \\ 121 | \min(2.0, 1.0 + \ln(\frac{c}{20})) & \text{if } c \geq 20 122 | \end{cases}$$ 123 | 124 | Where: 125 | - $c$ = total evaluation count (fooled + not fooled) 126 | - Reference count of 20 gives multiplier of 1.0 127 | - Sample sizes below 20 are penalized 128 | - Sample sizes above 20 receive logarithmic bonus up to 2.0x 129 | 130 | ### Final Generator Reward 131 | 132 | The total generator reward combines both components: 133 | 134 | $$R_{\text{total}} = R_{\text{base}} \cdot M$$ 135 | 136 | This design incentivizes generators to: 137 | 1. Produce high-quality, valid content (base reward) 138 | 2. Create adversarially robust content that can fool discriminators (multiplier) 139 | 3. Participate in more evaluations for sample size bonuses 140 | 141 | 142 | 143 | ## Discriminator Rewards 144 | 145 | The discriminator incentive mechanism uses a winner-take-all approach with a dynamic threshold system that gradually decays over time. This ensures that only the best-performing discriminators receive rewards while maintaining competition over extended periods. Currently, scores are mean of image and video multiclass MCCs. In the future, this may be broken out into individual image and video thresholds. 146 | 147 | ### Threshold Function 148 | 149 | The threshold function $T(x)$ is defined as: 150 | 151 | $$T(x) = \max \left( S + \varepsilon, (S + \text{boost}) e^{-kx} \right)$$ 152 | 153 | Where: 154 | - $S$ = new leader's score (e.g., 0.87) 155 | - $\varepsilon$ = floor margin (use 0.01 $\Rightarrow$ floor = $S + 0.01$) 156 | - $\text{boost} = \min(\text{cap}, g \cdot \Delta)$, with $\Delta = S - S_{\text{prev}}$ 157 | - pick $g = 2.5$ and $\text{cap} = 0.05$ 158 | - (so a +0.02 improvement $\Rightarrow 2.5 \times 0.02 = 0.05 \Rightarrow$ full 5-point boost) 159 | - We choose $k$ by duration (lands exactly on the floor at $H$ epochs): 160 | 161 | $$k = \frac{1}{H} \ln \left( \frac{S + \text{boost}}{S + \varepsilon} \right)$$ 162 | 163 | - with $H = 140$ epochs (~1 week) 164 | 165 | ### Example 166 | 167 | Using the scenario from the threshold calculation: 168 | - $S_{\text{prev}} = 0.85$, $S = 0.87$, $\Delta = 0.02$ 169 | - $\text{boost} = \min(0.05, 2.5 \times 0.02) = 0.05 \Rightarrow$ initial $T(0) = 0.92$ 170 | - $\varepsilon = 0.01 \Rightarrow$ floor $= 0.88$ 171 | - $k = \frac{1}{140} \ln(0.92/0.88) \approx 3.17 \times 10^{-4}$ 172 | - Then $T(x)$ decays smoothly: $\sim 0.900$ around 70 epochs, and clamps to 0.88 at 140. 173 | 174 | 175 | The following plot illustrates how the threshold function decays over time using the example parameters above: 176 | 177 | ![Threshold Decay Function](static/threshold_decay.png) -------------------------------------------------------------------------------- /gas/scraping/base.py: -------------------------------------------------------------------------------- 1 | import os 2 | import requests 3 | import time 4 | from abc import ABC, abstractmethod 5 | from PIL import Image 6 | from io import BytesIO 7 | import bittensor as bt 8 | import json 9 | 10 | from gas.types import Modality 11 | 12 | 13 | class BaseScraper(ABC): 14 | """ 15 | Base class for image scrapers with common functionality 16 | 17 | Parameters: 18 | ----------- 19 | min_width : int 20 | Minimum width for downloaded images (default: 128) 21 | min_height : int 22 | Minimum height for downloaded images (default: 128) 23 | media_type : MediaType, optional 24 | Type of media being scraped 25 | """ 26 | 27 | def __init__(self, min_width=128, min_height=128, media_type=None): 28 | self.min_width = min_width 29 | self.min_height = min_height 30 | self.media_type = media_type 31 | self.modality = Modality.IMAGE # does not support video yet 32 | 33 | def _check_image_size(self, url): 34 | """ 35 | Check if image meets minimum size requirements 36 | 37 | Parameters: 38 | ----------- 39 | url : str 40 | Image URL to check 41 | 42 | Returns: 43 | -------- 44 | tuple 45 | (meets_requirements: bool, width: int, height: int) 46 | """ 47 | try: 48 | response = requests.get(url, timeout=10, stream=True) 49 | if response.status_code == 200: 50 | img = Image.open(BytesIO(response.content)) 51 | width, height = img.size 52 | meets_requirements = width >= self.min_width and height >= self.min_height 53 | return meets_requirements, width, height 54 | else: 55 | bt.logging.warning(f"Failed to fetch image for size check: {url}") 56 | return False, 0, 0 57 | except Exception as e: 58 | bt.logging.warning(f"Error checking image size for {url}: {str(e)}") 59 | return False, 0, 0 60 | 61 | def _get_file_extension(self, url): 62 | """Get appropriate file extension based on content type""" 63 | extension = '.jpg' # Default 64 | try: 65 | response = requests.head(url, timeout=10) 66 | content_type = response.headers.get('content-type', '') 67 | 68 | if 'image/jpeg' in content_type: 69 | extension = '.jpg' 70 | elif 'image/png' in content_type: 71 | extension = '.png' 72 | elif 'image/gif' in content_type: 73 | extension = '.gif' 74 | elif 'image/webp' in content_type: 75 | extension = '.webp' 76 | except Exception as e: 77 | bt.logging.info(f"Error fetching headers for {url}: {str(e)}") 78 | 79 | return extension 80 | 81 | def _download_single_image(self, url, width=None, height=None): 82 | """ 83 | Download a single image and yield the image data 84 | 85 | Parameters: 86 | ----------- 87 | url : str 88 | Image URL to download 89 | width : int, optional 90 | Image width for logging 91 | height : int, optional 92 | Image height for logging 93 | 94 | Yields: 95 | ------- 96 | tuple 97 | (success: bool, image_data: PIL.Image or None, metadata: dict or None) 98 | """ 99 | try: 100 | image_response = requests.get(url, timeout=10) 101 | if image_response.status_code == 200: 102 | # Convert to PIL Image 103 | img = Image.open(BytesIO(image_response.content)) 104 | size_info = f" ({width}x{height})" if width and height else "" 105 | bt.logging.trace(f"Downloaded image from {url}{size_info}") 106 | 107 | metadata = { 108 | 'url': url, 109 | 'width': width, 110 | 'height': height, 111 | 'content_type': image_response.headers.get('content-type', ''), 112 | 'content_length': len(image_response.content) 113 | } 114 | 115 | return True, img, metadata 116 | else: 117 | bt.logging.error(f"Failed to download {url}, status code: {image_response.status_code}") 118 | return False, None, None 119 | except Exception as e: 120 | bt.logging.error(f"Error downloading image from {url}: {str(e)}") 121 | return False, None, None 122 | 123 | @abstractmethod 124 | def get_image_urls(self, queries, query_ids=None, limit=5): 125 | """ 126 | Abstract method to get image URLs - must be implemented by subclasses 127 | 128 | Parameters: 129 | ----------- 130 | queries : str or list 131 | Search query or list of queries 132 | query_ids: str or list 133 | Query ids for tracking 134 | limit : int 135 | Maximum number of images per query 136 | 137 | Returns: 138 | -------- 139 | dict 140 | Dictionary with query keys and lists of image data 141 | """ 142 | pass 143 | 144 | def download_images(self, queries=None, query_ids=None, urls=None, source_image_paths=None, limit=5): 145 | """ 146 | Download images based on queries with size constraints 147 | 148 | Parameters: 149 | ----------- 150 | queries : str or list 151 | Search query or list of queries (set if not using source_images or urls) 152 | query_ids: str or list 153 | Query id for tracking 154 | urls: str or list 155 | Pre-fetched image urls (set if not using source_images or queries) 156 | source_image_paths: str or list 157 | Local path to image(s) with which to perform reverse image search 158 | limit : int 159 | Maximum number of images to download per query 160 | 161 | Yields: 162 | ------- 163 | tuple 164 | (query_id: str, image_data: dict) where image_data contains: 165 | - url: str 166 | - image_content: PIL.Image 167 | - width: int 168 | - height: int 169 | - metadata: dict (query, source_url, title, source, etc.) 170 | """ 171 | if sum(x is not None for x in [queries, urls, source_image_paths]) != 1: 172 | raise ValueError("Either queries, urls, or source_image must be provided (mutually exclusive)") 173 | 174 | if urls is None: 175 | # Get more URLs than needed to account for size filtering 176 | image_urls = self.get_image_urls( 177 | queries=queries, 178 | query_ids=query_ids, 179 | source_image_paths=source_image_paths, 180 | limit=limit * 3 181 | ) 182 | 183 | for query_id in image_urls: 184 | downloaded_count = 0 185 | 186 | for img_data in image_urls[query_id]: 187 | if downloaded_count >= limit: 188 | break 189 | 190 | url = img_data['url'] 191 | 192 | meets_size_req, width, height = self._check_image_size(url) 193 | 194 | if not meets_size_req: 195 | bt.logging.trace(f"Skipping image {url} - size {width}x{height} below minimum {self.min_width}x{self.min_height}") 196 | continue 197 | 198 | # Download the image 199 | success, image_content, download_metadata = self._download_single_image(url, width, height) 200 | 201 | if not success or image_content is None: 202 | continue 203 | 204 | # Prepare metadata 205 | metadata = { 206 | 'query': img_data.get('query', query_id), 207 | 'source_url': img_data.get('source_url'), 208 | 'title': img_data.get('title', ''), 209 | 'source': img_data.get('source', 'unknown'), 210 | 'download_url': url, 211 | **(download_metadata or {}) 212 | } 213 | 214 | # Yield the downloaded image data 215 | image_data = { 216 | 'url': url, 217 | 'image_content': image_content, 218 | 'width': width, 219 | 'height': height, 220 | 'metadata': metadata 221 | } 222 | 223 | downloaded_count += 1 224 | yield query_id, image_data 225 | 226 | bt.logging.debug(f"Downloaded {downloaded_count} images for query {query_id}") -------------------------------------------------------------------------------- /gas/types.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from dataclasses import dataclass, field 3 | from enum import Enum, auto 4 | from pydantic import BaseModel 5 | from typing import Dict, List, Any, Optional, Union 6 | import json 7 | 8 | 9 | class NeuronType(Enum): 10 | VALIDATOR = "VALIDATOR" 11 | MINER = "MINER" 12 | 13 | 14 | class MinerType(Enum): 15 | DISCRIMINATOR = "DISCRIMINATOR" 16 | GENERATOR = "GENERATOR" 17 | 18 | 19 | class DiscriminatorType(Enum): 20 | DETECTOR = "DETECTOR" 21 | SEGMENTER = "SEGMENTER" 22 | 23 | 24 | class FileType(Enum): 25 | PARQUET = auto() 26 | ZIP = auto() 27 | VIDEO = auto() 28 | IMAGE = auto() 29 | 30 | 31 | class Modality(str, Enum): 32 | IMAGE = "image" 33 | VIDEO = "video" 34 | 35 | 36 | class MediaType(str, Enum): 37 | REAL = "real" 38 | SYNTHETIC = "synthetic" 39 | SEMISYNTHETIC = "semisynthetic" 40 | 41 | @property 42 | def int_value(self): 43 | """Get the integer value for this media type""" 44 | mapping = { 45 | MediaType.REAL: 0, 46 | MediaType.SYNTHETIC: 1, 47 | MediaType.SEMISYNTHETIC: 2, 48 | } 49 | return mapping[self] 50 | 51 | 52 | class SourceType(str, Enum): 53 | """Canonical source types for media provenance.""" 54 | 55 | SCRAPER = "scraper" 56 | DATASET = "dataset" 57 | GENERATED = "generated" 58 | MINER = "miner" 59 | 60 | 61 | SOURCE_TYPE_TO_NAME: Dict[SourceType, str] = { 62 | SourceType.GENERATED: "model_name", 63 | SourceType.DATASET: "dataset_name", 64 | SourceType.SCRAPER: "download_url", 65 | SourceType.MINER: "hotkey", 66 | } 67 | 68 | 69 | SOURCE_TYPE_TO_DB_NAME_FIELD: Dict[SourceType, str] = { 70 | SourceType.GENERATED: "model_name", 71 | SourceType.DATASET: "dataset_name", 72 | SourceType.SCRAPER: "scraper_name", 73 | SourceType.MINER: "hotkey", 74 | } 75 | 76 | 77 | @dataclass 78 | class DatasetConfig: 79 | """For datasets used by the Validator""" 80 | path: str # HuggingFace path 81 | modality: Modality 82 | media_type: MediaType 83 | file_format: str = "" 84 | source_format: str = "" 85 | priority: int = 1 # Optional: priority for sampling (higher is more frequent) 86 | enabled: bool = True 87 | 88 | def __post_init__(self): 89 | """Validate and set defaults""" 90 | if not self.source_format: 91 | if self.modality == Modality.IMAGE: 92 | self.source_format = "parquet" 93 | elif self.modality == Modality.VIDEO: 94 | self.source_format = "zip" 95 | 96 | if isinstance(self.modality, str): 97 | self.modality = Modality(self.modality.lower()) 98 | 99 | if isinstance(self.media_type, str): 100 | self.media_type = MediaType(self.media_type.lower()) 101 | 102 | 103 | class ModelTask(str, Enum): 104 | "Tasks supported by validator models" 105 | TEXT_TO_IMAGE = "t2i" 106 | TEXT_TO_VIDEO = "t2v" 107 | IMAGE_TO_IMAGE = "i2i" 108 | IMAGE_TO_VIDEO = "i2v" 109 | 110 | 111 | class ModelConfig: 112 | """ 113 | Configuration for a generative AI model. 114 | 115 | Attributes: 116 | path: The Hugging Face model path or identifier 117 | task: The primary task of the model (T2I, T2V, I2I) 118 | media_type: Type of output (synthetic or semisynthetic) 119 | pipeline_cls: Pipeline class used to load the model 120 | pretrained_args: Arguments for the from_pretrained method 121 | generation_args: Default arguments for generation 122 | tags: List of tags for categorizing the model 123 | use_autocast: Whether to use autocast during generation 124 | scheduler: Optional scheduler configuration 125 | scheduler_cls: Optional scheduler class 126 | scheduler_args: Optional scheduler args 127 | """ 128 | 129 | def __init__( 130 | self, 131 | path: str, 132 | task: ModelTask, 133 | pipeline_cls: Union[Any, Dict[str, Any]], 134 | media_type: Optional[MediaType] = None, 135 | pretrained_args: Dict[str, Any] = None, 136 | generation_args: Dict[str, Any] = None, 137 | tags: List[str] = None, 138 | use_autocast: bool = True, 139 | enable_model_cpu_offload: bool = False, 140 | enable_sequential_cpu_offload: bool = False, 141 | vae_enable_slicing: bool = False, 142 | vae_enable_tiling: bool = False, 143 | scheduler: Dict[str, Any] = None, 144 | save_args: Dict[str, Any] = None, 145 | pipeline_stages: List[Dict[str, Any]] = None, 146 | clear_memory_on_stage_end: bool = False, 147 | lora_model_id: str = None, 148 | lora_loading_args: Dict[str, Any] = None, 149 | ): 150 | self.path = path 151 | self.task = task 152 | self.pipeline_cls = pipeline_cls 153 | self.media_type = media_type 154 | 155 | if self.media_type is None: 156 | self.media_type = ( 157 | MediaType.SEMISYNTHETIC 158 | if task == ModelTask.IMAGE_TO_IMAGE 159 | else MediaType.SYNTHETIC 160 | ) 161 | 162 | self.pretrained_args = pretrained_args or {} 163 | self.generation_args = generation_args or {} 164 | self.tags = tags or [] 165 | self.use_autocast = use_autocast 166 | self.enable_model_cpu_offload = enable_model_cpu_offload 167 | self.enable_sequential_cpu_offload = enable_sequential_cpu_offload 168 | self.vae_enable_slicing = vae_enable_slicing 169 | self.vae_enable_tiling = vae_enable_tiling 170 | self.scheduler = scheduler 171 | self.save_args = save_args or {} 172 | self.pipeline_stages = pipeline_stages 173 | self.clear_memory_on_stage_end = clear_memory_on_stage_end 174 | self.lora_model_id = lora_model_id 175 | self.lora_loading_args = lora_loading_args 176 | 177 | def to_dict(self) -> Dict[str, Any]: 178 | """Convert config to dictionary format""" 179 | return { 180 | "pipeline_cls": self.pipeline_cls, 181 | "from_pretrained_args": self.pretrained_args, 182 | "generation_args": self.generation_args, 183 | "use_autocast": self.use_autocast, 184 | "enable_model_cpu_offload": self.enable_model_cpu_offload, 185 | "enable_sequential_cpu_offload": self.enable_sequential_cpu_offload, 186 | "vae_enable_slicing": self.vae_enable_slicing, 187 | "vae_enable_tiling": self.vae_enable_tiling, 188 | "scheduler": self.scheduler, 189 | "save_args": self.save_args, 190 | "pipeline_stages": self.pipeline_stages, 191 | "clear_memory_on_stage_end": self.clear_memory_on_stage_end, 192 | } 193 | 194 | 195 | # ============================================================================= 196 | # MODEL MANAGEMENT TYPES 197 | # ============================================================================= 198 | 199 | @dataclass 200 | class DiscriminatorModelId: 201 | """Unique identifier for a discriminator model""" 202 | 203 | key: str # Remote storage key 204 | hash: str # Model dir hash 205 | 206 | def __post_init__(self): 207 | """Validate and truncate commit and hash to 16 chars""" 208 | if self.hash and len(self.hash) > 16: 209 | self.hash = self.hash[:16] 210 | 211 | def to_compressed_str(self) -> str: 212 | """Convert to compressed string for chain storage""" 213 | data = { 214 | "key": self.key, 215 | "hash": self.hash, 216 | } 217 | return json.dumps(data, separators=(",", ":")) 218 | 219 | @classmethod 220 | def from_compressed_str(cls, compressed_str: str) -> "DiscriminatorModelId": 221 | """Create DiscriminatorModelId from compressed string""" 222 | data = json.loads(compressed_str) 223 | return cls( 224 | key=data["key"], 225 | hash=data["hash"], 226 | ) 227 | 228 | def __eq__(self, other): 229 | if not isinstance(other, DiscriminatorModelId): 230 | return False 231 | return ( 232 | self.key == other.key 233 | and self.hash == other.hash 234 | ) 235 | 236 | 237 | @dataclass 238 | class DiscriminatorModelMetadata: 239 | """Metadata for a discriminator model stored on chain""" 240 | 241 | id: DiscriminatorModelId 242 | block: int # Block number when metadata was stored 243 | 244 | def to_dict(self) -> dict: 245 | """Convert to dictionary""" 246 | return { 247 | "id": { 248 | "key": self.id.key, 249 | "hash": self.id.hash, 250 | }, 251 | "block": self.block, 252 | } 253 | 254 | @classmethod 255 | def from_dict(cls, data: dict) -> "DiscriminatorModelMetadata": 256 | """Create DiscriminatorModelMetadata from dictionary""" 257 | model_id = DiscriminatorModelId( 258 | key=data["id"]["key"], 259 | hash=data["id"]["hash"], 260 | ) 261 | return cls(id=model_id, block=data["block"]) 262 | 263 | 264 | class ValidatorConfig(BaseModel): 265 | skip_weight_set: Optional[bool] = False 266 | set_weights_on_start: Optional[bool] = False 267 | max_concurrent_organics: Optional[int] = 2 268 | -------------------------------------------------------------------------------- /gas/generation/util/image.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import PIL 3 | import os 4 | from PIL import Image, ImageDraw 5 | from typing import Tuple, Union, List 6 | 7 | 8 | def resize_image( 9 | image: PIL.Image.Image, max_width: int, max_height: int 10 | ) -> PIL.Image.Image: 11 | """Resize the image to fit within specified dimensions while maintaining aspect ratio.""" 12 | original_width, original_height = image.size 13 | 14 | # Calculate the aspect ratio and determine new dimensions 15 | aspect_ratio = original_width / original_height 16 | new_width = min(max_width, original_width) 17 | new_height = int(new_width / aspect_ratio) 18 | 19 | if new_height > max_height: 20 | new_height = max_height 21 | new_width = int(new_height * aspect_ratio) 22 | 23 | # Resize the image using the high-quality LANCZOS filter 24 | resized_image = image.resize((new_width, new_height), PIL.Image.LANCZOS) 25 | return resized_image 26 | 27 | 28 | def resize_images_in_directory(directory, target_width, target_height): 29 | """ 30 | Resize all images in the specified directory to the target width and height. 31 | 32 | Args: 33 | directory (str): Path to the directory containing images. 34 | target_width (int): Target width for resizing the images. 35 | target_height (int): Target height for resizing the images. 36 | """ 37 | # List all files in the directory 38 | for filename in os.listdir(directory): 39 | if filename.endswith( 40 | (".png", ".jpg", ".jpeg", ".bmp", ".gif") 41 | ): # Check for image file extensions 42 | filepath = os.path.join(directory, filename) 43 | with PIL.Image.open(filepath) as img: 44 | # Resize the image and save back to the file location 45 | resized_img = resize_image( 46 | img, max_width=target_width, max_height=target_height 47 | ) 48 | resized_img.save(filepath) 49 | 50 | 51 | def save_images_to_disk( 52 | image_dataset, start_index, num_images, save_directory, resize=True 53 | ): 54 | if not os.path.exists(save_directory): 55 | os.makedirs(save_directory) 56 | 57 | for i in range(start_index, start_index + num_images): 58 | try: 59 | image_data = image_dataset[i] # Retrieve image using the __getitem__ method 60 | image = image_data["image"] # Extract the image 61 | image_id = image_data["id"] # Extract the image ID 62 | file_path = os.path.join( 63 | save_directory, f"{image_id}.jpg" 64 | ) # Construct file path 65 | # if resize: 66 | # image = resize_image(image, TARGET_IMAGE_SIZE[0], TARGET_IMAGE_SIZE[1]) 67 | image.save(file_path, "JPEG") # Save the image 68 | print(f"Saved: {file_path}") 69 | except Exception as e: 70 | print(f"Failed to save image {i}: {e}") 71 | 72 | 73 | def ensure_mask_3d(mask: np.ndarray) -> np.ndarray: 74 | """ 75 | Ensure the mask is 3D (H, W, 1) if it's 2D (H, W). 76 | """ 77 | if mask.ndim == 2: 78 | return mask[:, :, None] 79 | return mask 80 | 81 | 82 | def create_random_mask( 83 | size: Tuple[int, int], 84 | min_size_ratio: float = 0.15, 85 | max_size_ratio: float = 0.5, 86 | allow_multiple: bool = True, 87 | allowed_shapes: list = ["rectangle", "circle", "ellipse", "triangle"], 88 | ) -> "Image.Image": 89 | """ 90 | Create a random mask (or masks) for i2i/inpainting with more variety. 91 | Returns a single-channel ("L" mode) mask image. 92 | """ 93 | w, h = size 94 | allowed_shapes = [s for s in allowed_shapes] 95 | max_retries = 5 96 | for attempt in range(max_retries): 97 | mask = Image.new("L", size, 0) 98 | draw = ImageDraw.Draw(mask) 99 | n_masks = np.random.randint(1, 5) if allow_multiple else 1 100 | for _ in range(n_masks): 101 | shape = np.random.choice(allowed_shapes) 102 | min_dim = min(w, h) 103 | min_pixel_size = 64 104 | min_mask_size = max(int(min_size_ratio * min_dim), min_pixel_size) 105 | max_mask_size = max(int(max_size_ratio * min_dim), min_pixel_size) 106 | if min_mask_size >= max_mask_size: 107 | width = min_mask_size 108 | height = min_mask_size 109 | else: 110 | width = np.random.randint(min_mask_size, max_mask_size) 111 | height = np.random.randint(min_mask_size, max_mask_size) 112 | width = min(width, w) 113 | height = min(height, h) 114 | if shape == "circle": 115 | r = min(width, height) // 2 116 | if r < 1: 117 | r = 1 118 | cx = np.random.randint(r, w - r + 1) 119 | cy = np.random.randint(r, h - r + 1) 120 | draw.ellipse([cx - r, cy - r, cx + r, cy + r], fill=255) 121 | elif shape == "rectangle": 122 | x = np.random.randint(0, w - width + 1) 123 | y = np.random.randint(0, h - height + 1) 124 | draw.rectangle([x, y, x + width, y + height], fill=255) 125 | elif shape == "ellipse": 126 | x = np.random.randint(0, w - width + 1) 127 | y = np.random.randint(0, h - height + 1) 128 | x0, y0, x1, y1 = x, y, x + width, y + height 129 | draw.ellipse([x0, y0, x1, y1], fill=255) 130 | elif shape == "triangle": 131 | min_triangle_size = max(96, min_mask_size) 132 | max_triangle_size = max(128, max_mask_size) 133 | min_triangle_size = min(min_triangle_size, w, h) 134 | max_triangle_size = min(max_triangle_size, w, h) 135 | if min_triangle_size >= max_triangle_size: 136 | width = max_triangle_size 137 | height = max_triangle_size 138 | else: 139 | width = np.random.randint(min_triangle_size, max_triangle_size) 140 | height = np.random.randint(min_triangle_size, max_triangle_size) 141 | x = np.random.randint(0, w - width + 1) 142 | y = np.random.randint(0, h - height + 1) 143 | jitter = lambda v, maxv: max( 144 | 0, 145 | min(v + np.random.randint(-width // 10, width // 10 + 1), maxv - 1), 146 | ) 147 | pt1 = (jitter(x, w), jitter(y, h)) 148 | pt2 = (jitter(x + width - 1, w), jitter(y, h)) 149 | pt3 = (jitter(x, w), jitter(y + height - 1, h)) 150 | pts = [pt1, pt2, pt3] 151 | draw.polygon(pts, fill=255) 152 | if np.array(mask).max() > 0: 153 | return mask 154 | return mask 155 | 156 | 157 | def is_black_output( 158 | modality: str, output: Union[List[Image.Image], Image.Image], threshold: int = 10 159 | ) -> bool: 160 | """ 161 | Returns True if the image or frames are (almost) completely black. 162 | """ 163 | try: 164 | if modality == "image": 165 | # Handle different image output formats 166 | if hasattr(output, "images"): 167 | arr = np.array(output.images[0]) 168 | elif isinstance(output, Image.Image): 169 | arr = np.array(output) 170 | else: 171 | arr = np.array(output) 172 | return np.mean(arr) < threshold 173 | elif modality == "video": 174 | # Handle different video output formats 175 | if hasattr(output, "frames"): 176 | frames = output.frames[0] 177 | else: 178 | frames = output 179 | 180 | # Check first few frames to avoid processing all frames unnecessarily 181 | sample_frames = frames[: min(5, len(frames))] if len(frames) > 5 else frames 182 | 183 | for frame in sample_frames: 184 | try: 185 | # Convert frame to numpy array if it's a PIL Image 186 | if hasattr(frame, "mode"): # PIL Image 187 | arr = np.array(frame) 188 | elif hasattr(frame, "numpy"): # Tensor 189 | arr = ( 190 | frame.numpy() 191 | if hasattr(frame, "numpy") 192 | else np.array(frame) 193 | ) 194 | else: 195 | arr = np.array(frame) 196 | 197 | # Ensure we have numeric data 198 | if arr.dtype.kind not in [ 199 | "i", 200 | "u", 201 | "f", 202 | ]: # not integer, unsigned, or float 203 | continue # Skip non-numeric frames 204 | 205 | # If any frame has reasonable brightness, it's not black 206 | if np.mean(arr) >= threshold: 207 | return False 208 | except Exception: 209 | # If we can't process a frame, assume it's not black 210 | continue 211 | 212 | return True # All processable frames are black 213 | except Exception: 214 | # If we can't analyze the output, assume it's not black to avoid false positives 215 | return False 216 | -------------------------------------------------------------------------------- /gas/verification/c2pa_verification.py: -------------------------------------------------------------------------------- 1 | """ 2 | C2PA Content Credentials verification for miner-submitted media. 3 | 4 | Validates that content has authentic provenance from trusted AI generators 5 | like OpenAI (DALL-E), Google (Gemini/Imagen), Adobe Firefly, etc. 6 | """ 7 | 8 | import io 9 | import tempfile 10 | from pathlib import Path 11 | from typing import Optional, Dict, Any, List, Union 12 | 13 | import bittensor as bt 14 | 15 | try: 16 | import c2pa 17 | C2PA_AVAILABLE = True 18 | except ImportError: 19 | C2PA_AVAILABLE = False 20 | bt.logging.warning("c2pa-python not installed. C2PA verification will be disabled.") 21 | 22 | 23 | # Known trusted AI generator issuers 24 | # These are organizations that embed C2PA credentials in their AI-generated content 25 | TRUSTED_ISSUERS = [ 26 | # OpenAI 27 | "openai", "openai.com", "dall-e", "sora", "chatgpt", 28 | 29 | # Google 30 | "google", "google.com", "google.llc", 31 | "imagen", "veo", "gemini", "deepmind", 32 | 33 | # Adobe 34 | "adobe", "adobe.com", "firefly", "contentauthenticity", 35 | 36 | # Microsoft 37 | "microsoft", "microsoft.com", "bing", "designer", "copilot", 38 | 39 | # Meta 40 | "meta", "meta.com", "facebook", "instagram", 41 | 42 | # Other 43 | "runway", "runwayml", "runwayml.com", 44 | "stability", "stability.ai", 45 | "pika", "pika.art", 46 | "canva", "canva.com", 47 | "shutterstock", "shutterstock.com", 48 | ] 49 | 50 | 51 | 52 | class C2PAVerificationResult: 53 | """Result of C2PA verification.""" 54 | 55 | def __init__( 56 | self, 57 | verified: bool = False, 58 | issuer: Optional[str] = None, 59 | is_trusted_issuer: bool = False, 60 | ai_generated: bool = False, 61 | manifest_data: Optional[Dict[str, Any]] = None, 62 | error: Optional[str] = None, 63 | ): 64 | self.verified = verified 65 | self.issuer = issuer 66 | self.is_trusted_issuer = is_trusted_issuer 67 | self.ai_generated = ai_generated 68 | self.manifest_data = manifest_data or {} 69 | self.error = error 70 | 71 | def to_dict(self) -> Dict[str, Any]: 72 | return { 73 | "verified": self.verified, 74 | "issuer": self.issuer, 75 | "is_trusted_issuer": self.is_trusted_issuer, 76 | "ai_generated": self.ai_generated, 77 | "error": self.error, 78 | } 79 | 80 | 81 | def verify_c2pa( 82 | media_data: Union[bytes, str, Path], 83 | trusted_issuers: Optional[List[str]] = None, 84 | ) -> C2PAVerificationResult: 85 | """ 86 | Verify C2PA content credentials in media. 87 | 88 | Args: 89 | media_data: Media as bytes or file path 90 | trusted_issuers: List of trusted issuer patterns (default: TRUSTED_ISSUERS) 91 | 92 | Returns: 93 | C2PAVerificationResult with verification details 94 | """ 95 | if not C2PA_AVAILABLE: 96 | return C2PAVerificationResult( 97 | verified=False, 98 | error="c2pa-python library not installed" 99 | ) 100 | 101 | if trusted_issuers is None: 102 | trusted_issuers = TRUSTED_ISSUERS 103 | 104 | temp_file = None 105 | try: 106 | # Handle bytes input by writing to temp file 107 | if isinstance(media_data, bytes): 108 | # Detect format from magic bytes 109 | suffix = _detect_format(media_data) 110 | temp_file = tempfile.NamedTemporaryFile(suffix=suffix, delete=False) 111 | temp_file.write(media_data) 112 | temp_file.close() 113 | file_path = temp_file.name 114 | else: 115 | file_path = str(media_data) 116 | 117 | # Read C2PA manifest from file 118 | try: 119 | with c2pa.Reader(file_path) as reader: 120 | manifest_json = reader.json() 121 | except Exception as e: 122 | # No C2PA manifest found - this is common for most images 123 | return C2PAVerificationResult( 124 | verified=False, 125 | error=f"No C2PA manifest found: {str(e)}" 126 | ) 127 | 128 | if not manifest_json: 129 | return C2PAVerificationResult( 130 | verified=False, 131 | error="Empty C2PA manifest" 132 | ) 133 | 134 | # Parse manifest to extract issuer and AI generation info 135 | issuer = _extract_issuer(manifest_json) 136 | ai_generated = _check_ai_generated(manifest_json) 137 | is_trusted = _is_trusted_issuer(issuer, trusted_issuers) 138 | 139 | return C2PAVerificationResult( 140 | verified=True, 141 | issuer=issuer, 142 | is_trusted_issuer=is_trusted, 143 | ai_generated=ai_generated, 144 | manifest_data={"raw": manifest_json}, 145 | ) 146 | 147 | except Exception as e: 148 | bt.logging.warning(f"C2PA verification error: {e}") 149 | return C2PAVerificationResult( 150 | verified=False, 151 | error=str(e) 152 | ) 153 | finally: 154 | # Clean up temp file 155 | if temp_file: 156 | try: 157 | Path(temp_file.name).unlink(missing_ok=True) 158 | except Exception: 159 | pass 160 | 161 | 162 | def _detect_format(data: bytes) -> str: 163 | """Detect image/video format from magic bytes.""" 164 | if data[:8] == b'\x89PNG\r\n\x1a\n': 165 | return ".png" 166 | elif data[:2] == b'\xff\xd8': 167 | return ".jpg" 168 | elif data[:4] == b'RIFF' and data[8:12] == b'WEBP': 169 | return ".webp" 170 | elif data[:4] == b'\x00\x00\x00\x1c' or data[:4] == b'\x00\x00\x00\x20': 171 | return ".mp4" 172 | elif data[:4] == b'\x1a\x45\xdf\xa3': 173 | return ".webm" 174 | else: 175 | return ".bin" 176 | 177 | 178 | def _extract_issuer(manifest_json: str) -> Optional[str]: 179 | """Extract issuer/claim generator from C2PA manifest.""" 180 | import json 181 | try: 182 | data = json.loads(manifest_json) if isinstance(manifest_json, str) else manifest_json 183 | 184 | # Check manifests for claim_generator or issuer 185 | manifests = data.get("manifests", {}) 186 | for manifest_id, manifest in manifests.items(): 187 | # Check claim_generator field 188 | claim_generator = manifest.get("claim_generator") 189 | if claim_generator: 190 | return claim_generator 191 | 192 | # Check signature info 193 | signature_info = manifest.get("signature_info", {}) 194 | issuer = signature_info.get("issuer") 195 | if issuer: 196 | return issuer 197 | 198 | # Check active manifest 199 | active_manifest = data.get("active_manifest") 200 | if active_manifest and active_manifest in manifests: 201 | manifest = manifests[active_manifest] 202 | return manifest.get("claim_generator") 203 | 204 | return None 205 | except Exception as e: 206 | bt.logging.debug(f"Error extracting issuer: {e}") 207 | return None 208 | 209 | 210 | def _check_ai_generated(manifest_json: str) -> bool: 211 | """Check if manifest indicates AI-generated content.""" 212 | import json 213 | try: 214 | data = json.loads(manifest_json) if isinstance(manifest_json, str) else manifest_json 215 | 216 | manifests = data.get("manifests", {}) 217 | for manifest_id, manifest in manifests.items(): 218 | # Check assertions for AI generation indicators 219 | assertions = manifest.get("assertions", []) 220 | for assertion in assertions: 221 | label = assertion.get("label", "") 222 | 223 | # C2PA uses specific labels for AI-generated content 224 | if "c2pa.ai_generated" in label: 225 | return True 226 | if "c2pa.ai" in label: 227 | return True 228 | 229 | # Check action data 230 | if assertion.get("data", {}).get("digitalSourceType") == "trainedAlgorithmicMedia": 231 | return True 232 | if assertion.get("data", {}).get("digitalSourceType") == "compositeWithTrainedAlgorithmicMedia": 233 | return True 234 | 235 | return False 236 | except Exception as e: 237 | bt.logging.debug(f"Error checking AI generation: {e}") 238 | return False 239 | 240 | 241 | def _is_trusted_issuer(issuer: Optional[str], trusted_issuers: List[str]) -> bool: 242 | """Check if issuer matches any trusted issuer pattern.""" 243 | if not issuer: 244 | return False 245 | 246 | issuer_lower = issuer.lower() 247 | for trusted in trusted_issuers: 248 | if trusted.lower() in issuer_lower: 249 | return True 250 | return False 251 | 252 | 253 | def is_from_trusted_generator( 254 | media_data: Union[bytes, str, Path], 255 | require_ai_label: bool = False, 256 | ) -> bool: 257 | """ 258 | Quick check if media is from a trusted AI generator. 259 | 260 | Args: 261 | media_data: Media as bytes or file path 262 | require_ai_label: If True, also require AI generation assertion 263 | 264 | Returns: 265 | True if from trusted generator with valid C2PA credentials 266 | """ 267 | result = verify_c2pa(media_data) 268 | 269 | if not result.verified: 270 | return False 271 | 272 | if not result.is_trusted_issuer: 273 | return False 274 | 275 | if require_ai_label and not result.ai_generated: 276 | return False 277 | 278 | return True 279 | 280 | -------------------------------------------------------------------------------- /gas/utils/autoupdater.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # Copyright © 2023 Yuma Rao 3 | # Copyright © 2024 Manifold Labs 4 | # Copyright © 2025 BitMind 5 | 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 7 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation 8 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 9 | # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 10 | 11 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of 12 | # the Software. 13 | 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 15 | # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 16 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 17 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 18 | # DEALINGS IN THE SOFTWARE. 19 | 20 | import signal 21 | import time 22 | import os 23 | import requests 24 | import subprocess 25 | import json 26 | import bittensor as bt 27 | import gas 28 | 29 | 30 | def get_running_sn34_apps_via_pm2(): 31 | """ 32 | Query PM2 for running processes and return names starting with 'sn34-'. 33 | """ 34 | try: 35 | result = subprocess.run([ 36 | "pm2", "jlist" 37 | ], capture_output=True, text=True) 38 | if result.returncode != 0: 39 | bt.logging.warning(f"pm2 jlist failed: {result.stderr}") 40 | return [] 41 | apps = json.loads(result.stdout) 42 | if not isinstance(apps, list): 43 | return [] 44 | return [proc.get('name') for proc in apps if isinstance(proc, dict) and isinstance(proc.get('name'), str) and proc.get('name').startswith('sn34-')] 45 | except Exception as e: 46 | bt.logging.warning(f"Failed to parse pm2 jlist: {e}") 47 | return [] 48 | 49 | 50 | def restart_pm2_services(base_path): 51 | """ 52 | Restart running sn34-* PM2 services only. Does not restart other processes. 53 | """ 54 | app_names = get_running_sn34_apps_via_pm2() 55 | 56 | if not app_names: 57 | bt.logging.warning("No sn34-* apps found; skipping PM2 restarts to avoid affecting other processes") 58 | return False 59 | 60 | # Restart each service individually 61 | success = True 62 | for app_name in app_names: 63 | try: 64 | bt.logging.info(f"Restarting {app_name}...") 65 | subprocess.run(["pm2", "restart", app_name], check=True) 66 | bt.logging.info(f"Successfully restarted {app_name}") 67 | except subprocess.CalledProcessError as e: 68 | bt.logging.error(f"Failed to restart {app_name}: {e}") 69 | success = False 70 | 71 | return success 72 | 73 | 74 | def run_gascli_install(base_path: str, install_type: str = "py-deps", clear_venv: bool = True) -> bool: 75 | """ 76 | Run gascli installation commands. 77 | 78 | Args: 79 | base_path: Path to the project root 80 | install_type: Type of installation ("py-deps", "sys-deps", or "full") 81 | clear_venv: Whether to clear existing .venv directory (default False for safe operation) 82 | 83 | Returns: 84 | bool: True if installation succeeded, False otherwise 85 | """ 86 | # Determine the correct gascli command 87 | if install_type == "py-deps": 88 | cmd_args = ["install-py-deps"] 89 | elif install_type == "sys-deps": 90 | cmd_args = ["install-sys-deps"] 91 | elif install_type == "full": 92 | cmd_args = ["install"] 93 | else: 94 | bt.logging.error(f"Invalid install_type: {install_type}") 95 | return False 96 | 97 | # Add clear-venv flag if requested (default is to preserve) 98 | if clear_venv: 99 | cmd_args.append("--clear-venv") 100 | 101 | # Find gascli executable 102 | venv_gascli = os.path.join(base_path, ".venv", "bin", "gascli") 103 | gascli_cmd = venv_gascli if os.path.exists(venv_gascli) else "gascli" 104 | 105 | try: 106 | bt.logging.info(f"Running gascli {' '.join(cmd_args)}...") 107 | result = subprocess.run( 108 | [gascli_cmd] + cmd_args, 109 | cwd=base_path, 110 | capture_output=True, 111 | text=True, 112 | timeout=600 # 10 minute timeout 113 | ) 114 | 115 | if result.returncode == 0: 116 | bt.logging.info(f"gascli {install_type} installation completed successfully") 117 | return True 118 | else: 119 | bt.logging.error(f"gascli {install_type} installation failed with return code {result.returncode}") 120 | if result.stdout: 121 | bt.logging.error(f"stdout: {result.stdout}") 122 | if result.stderr: 123 | bt.logging.error(f"stderr: {result.stderr}") 124 | return False 125 | 126 | except subprocess.TimeoutExpired: 127 | bt.logging.error(f"gascli {install_type} installation timed out") 128 | return False 129 | except Exception as e: 130 | bt.logging.error(f"Error running gascli {install_type} installation: {e}") 131 | return False 132 | 133 | 134 | def autoupdate(branch: str = "main", force=False, install_deps: bool = False, install_type: str = "py-deps", clear_venv: bool = True): 135 | """ 136 | Automatically updates the codebase to the latest version available on the specified branch. 137 | 138 | This function checks the remote repository for the latest version by fetching the VERSION file from the specified branch. 139 | If the local version is older than the remote version, it performs a git pull to update the local codebase to the latest version. 140 | After successfully updating, it restarts the application with the updated code. 141 | 142 | Args: 143 | - branch (str): The name of the branch to check for updates. Defaults to "main". 144 | - force (bool): Force update even if versions are the same. Defaults to False. 145 | - install_deps (bool): Whether to run dependency installation after update. Defaults to False. 146 | - install_type (str): Type of installation to run ("py-deps", "sys-deps", or "full"). Defaults to "py-deps". 147 | - clear_venv (bool): Whether to clear existing .venv directory during installation. Defaults to True. 148 | 149 | Note: 150 | - The function assumes that the local codebase is a git repository and has the same structure as the remote repository. 151 | - It requires git to be installed and accessible from the command line. 152 | - The function will restart the application using PM2 services defined in validator.config.js. 153 | - If install_deps is True, it will run gascli installation commands after update but before restart. 154 | - When install_deps is True, the existing .venv is cleared by default during installation to ensure a clean environment. 155 | - If the update fails, manual intervention is required to resolve the issue and restart the application. 156 | """ 157 | bt.logging.info("Checking for updates...") 158 | try: 159 | github_url = f"https://raw.githubusercontent.com/BitMind-AI/bitmind-subnet/{branch}/VERSION?ts={time.time()}" 160 | response = requests.get( 161 | github_url, 162 | headers={ 163 | "Cache-Control": "no-cache, no-store, must-revalidate", 164 | "Pragma": "no-cache", 165 | "Expires": "0" 166 | }, 167 | ) 168 | response.raise_for_status() 169 | repo_version = response.content.decode().strip() 170 | latest_version = tuple(map(int, repo_version.split("."))) 171 | local_version = tuple(map(int, gas.__version__.split("."))) 172 | 173 | bt.logging.info(f"Local version: {local_version}") 174 | bt.logging.info(f"Latest version: {latest_version}") 175 | 176 | if latest_version > local_version or force: 177 | bt.logging.info(f"A newer version is available. Updating...") 178 | base_path = os.path.abspath(__file__) 179 | while os.path.basename(base_path) != "bitmind-subnet": 180 | base_path = os.path.dirname(base_path) 181 | 182 | os.system(f"cd {base_path} && git pull") 183 | 184 | with open(os.path.join(base_path, "VERSION")) as f: 185 | new_version = f.read().strip() 186 | new_version = tuple(map(int, new_version.split("."))) 187 | 188 | if new_version == latest_version: 189 | bt.logging.info("Updated successfully.") 190 | 191 | # Install dependencies if requested 192 | if install_deps: 193 | dependency_install_success = run_gascli_install(base_path, install_type, clear_venv) 194 | if not dependency_install_success: 195 | bt.logging.warning("Dependency installation failed, but continuing with restart...") 196 | 197 | # Restart PM2 services using validator.config.js (with robust fallbacks) 198 | if restart_pm2_services(base_path): 199 | bt.logging.info("All PM2 services restarted successfully.") 200 | bt.logging.info(f"Restarting validator") 201 | os.kill(os.getpid(), signal.SIGINT) 202 | else: 203 | bt.logging.error("Failed to restart some PM2 services. Manual restart may be required.") 204 | else: 205 | bt.logging.error("Update failed. Manual update required.") 206 | except Exception as e: 207 | bt.logging.error(f"Update check failed: {e}") --------------------------------------------------------------------------------