├── tests ├── __init__.py ├── models │ ├── __init__.py │ ├── test_token_identifier.py │ ├── test_token_metadata.py │ └── test_token.py ├── resolver │ ├── __init__.py │ ├── sample_files │ │ └── bayc.json │ ├── test_rarity_sniper.py │ ├── test_rarity_sniffer.py │ ├── test_trait_sniper.py │ ├── test_external_rarity_provider.py │ └── test_testset_resolver.py ├── scoring │ ├── __init__.py │ ├── test_token_feature_extractor.py │ ├── test_scorer.py │ ├── test_scoring_handlers.py │ └── test_utils.py ├── conftest.py ├── test_rarity_ranker.py └── helpers.py ├── open_rarity ├── data │ ├── __init__.py │ └── test_collections.json ├── resolver │ ├── __init__.py │ ├── models │ │ ├── __init__.py │ │ ├── collection_with_metadata.py │ │ └── token_with_rarity_data.py │ └── rarity_providers │ │ ├── __init__.py │ │ ├── rank_resolver.py │ │ ├── rarity_sniper.py │ │ ├── rarity_sniffer.py │ │ └── trait_sniper.py ├── models │ ├── utils │ │ ├── __init__.py │ │ └── attribute_utils.py │ ├── chain.py │ ├── __init__.py │ ├── token_ranking_features.py │ ├── token_standard.py │ ├── token_rarity.py │ ├── token_identifier.py │ ├── token.py │ ├── token_metadata.py │ └── collection.py ├── scoring │ ├── handlers │ │ ├── __init__.py │ │ ├── scoring_handlers.md │ │ ├── sum_scoring_handler.py │ │ ├── arithmetic_mean_scoring_handler.py │ │ ├── harmonic_mean_scoring_handler.py │ │ ├── geometric_mean_scoring_handler.py │ │ └── information_content_scoring_handler.py │ ├── __init__.py │ ├── token_feature_extractor.py │ ├── scoring_handler.py │ ├── utils.py │ └── scorer.py ├── __init__.py └── rarity_ranker.py ├── img └── OR_Github_banner.jpg ├── .coveragerc ├── cached_data └── cached_data.md ├── .flake8 ├── noxfile.py ├── .github ├── workflows │ ├── tests.yaml │ └── code-quality.yaml └── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md ├── icons └── OpenRarity_Icons │ ├── 03Black │ ├── Color 24x24-Black.svg │ ├── Color 20x20-Black.svg │ ├── Color 14x14-Black.svg │ └── Color 16x16-Black.svg │ ├── 04White │ ├── Color 24x24-White.svg │ ├── Color 20x20-White.svg │ ├── Color 14x14-White.svg │ └── Color 16x16-White.svg │ ├── 01Color_LightMode │ ├── Color 24x24-LightMode.svg │ ├── Color 20x20-LightMode.svg │ ├── Color 14x14-LightMode.svg │ └── Color 16x16-LightMode.svg │ └── 02Color_DarkMode │ ├── Color 24x24-DarkMode.svg │ ├── Color 20x20-DarkMode.svg │ ├── Color 14x14-DarkMode.svg │ └── Color 16x16-DarkMode.svg ├── .gitignore ├── CONTRIBUTING.md ├── .pre-commit-config.yaml ├── pyproject.toml ├── scripts ├── score_generated_collection.py └── score_real_collections.py ├── README.md ├── CODE_OF_CONDUCT.md └── LICENSE /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /open_rarity/data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/resolver/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/scoring/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /open_rarity/resolver/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /open_rarity/models/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /open_rarity/resolver/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /open_rarity/scoring/handlers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /open_rarity/resolver/rarity_providers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /open_rarity/scoring/__init__.py: -------------------------------------------------------------------------------- 1 | from .scorer import Scorer 2 | -------------------------------------------------------------------------------- /img/OR_Github_banner.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRarity/open-rarity/HEAD/img/OR_Github_banner.jpg -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = tests/* 3 | # exclude resolver from coverage 4 | open_rarity/resolver/* 5 | -------------------------------------------------------------------------------- /tests/resolver/sample_files/bayc.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "collection_name": "Bored Ape Yacht Club", 4 | "collection_slug": "boredapeyachtclub" 5 | } 6 | ] 7 | -------------------------------------------------------------------------------- /open_rarity/models/chain.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from enum import Enum 3 | 4 | 5 | @dataclass 6 | class Chain(Enum): 7 | """ENUM represents the blockchain.""" 8 | 9 | ETH = 1 10 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | def pytest_addoption(parser): 2 | parser.addoption( 3 | "--run-resolvers", 4 | action="store_true", 5 | default=False, 6 | help="Run slow resolver tests", 7 | ) 8 | -------------------------------------------------------------------------------- /cached_data/cached_data.md: -------------------------------------------------------------------------------- 1 | # About 2 | This folder contains sample cached data that can be used to more efficiency run scripts score_generated_collection or 3 | the resolver to generate OpenRarity and/or external rarity ranks. 4 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 88 3 | extend-ignore = E203 4 | exclude = 5 | __init__.py 6 | __pycache__ 7 | settings.py 8 | env 9 | .env 10 | .venv 11 | .github 12 | .pytest_cache 13 | -------------------------------------------------------------------------------- /open_rarity/scoring/handlers/scoring_handlers.md: -------------------------------------------------------------------------------- 1 | # Scoring Handlers 2 | Handlers are algorithm-specific where each handler has a different formula for calculating scoring. Each handler must be implicitly using the ScoringHandler interface 3 | -------------------------------------------------------------------------------- /open_rarity/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .collection import Collection 2 | from .token import Token 3 | from .token_identifier import EVMContractTokenIdentifier 4 | from .token_metadata import StringAttribute, TokenMetadata 5 | from .token_rarity import TokenRarity 6 | from .token_standard import TokenStandard 7 | -------------------------------------------------------------------------------- /open_rarity/__init__.py: -------------------------------------------------------------------------------- 1 | from .models import ( 2 | Collection, 3 | EVMContractTokenIdentifier, 4 | StringAttribute, 5 | Token, 6 | TokenMetadata, 7 | TokenRarity, 8 | TokenStandard, 9 | ) 10 | from .rarity_ranker import RarityRanker 11 | from .scoring import Scorer as OpenRarityScorer 12 | -------------------------------------------------------------------------------- /open_rarity/resolver/rarity_providers/rank_resolver.py: -------------------------------------------------------------------------------- 1 | from abc import abstractmethod 2 | from typing import Protocol, runtime_checkable 3 | 4 | 5 | @runtime_checkable 6 | class RankResolver(Protocol): 7 | @staticmethod 8 | @abstractmethod 9 | def get_all_ranks(contract_address_or_slug: str) -> dict[str, int]: 10 | raise NotImplementedError 11 | -------------------------------------------------------------------------------- /noxfile.py: -------------------------------------------------------------------------------- 1 | import nox 2 | from nox import Session 3 | 4 | 5 | @nox.session(python=["3.10", "3.11"], reuse_venv=True) 6 | @nox.parametrize("pydantic", ["1.9.1", "2.0.3"]) 7 | def tests(session: Session, pydantic): 8 | session.install("poetry") 9 | session.run("poetry", "install") 10 | session.install(f"pydantic=={pydantic}") 11 | session.run("pytest") 12 | -------------------------------------------------------------------------------- /.github/workflows/tests.yaml: -------------------------------------------------------------------------------- 1 | name: Unit tests 2 | on: [push] 3 | jobs: 4 | unit-tests: 5 | runs-on: ubuntu-latest 6 | 7 | steps: 8 | - uses: actions/checkout@v2 9 | 10 | - name: Install Nox 11 | uses: wntrblm/nox@2023.04.22 12 | with: 13 | python-versions: "3.10,3.11" 14 | 15 | - name: Unit tests 16 | run: nox 17 | -------------------------------------------------------------------------------- /open_rarity/models/token_ranking_features.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | 4 | @dataclass 5 | class TokenRankingFeatures: 6 | """Class represents all standardized ranking features 7 | that should be considered by the ranking function. 8 | 9 | Attributes 10 | ---------- 11 | unique_attribute_count : int 12 | count of unique attributes in the token 13 | """ 14 | 15 | unique_attribute_count: int 16 | -------------------------------------------------------------------------------- /open_rarity/resolver/models/collection_with_metadata.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | from open_rarity.models.collection import Collection 4 | 5 | 6 | @dataclass 7 | class CollectionWithMetadata: 8 | # This class is just used for resolving different rarity ranks from 9 | # various providers and allowing us to compare across rarity providers. 10 | collection: Collection 11 | contract_addresses: list[str] 12 | token_total_supply: int 13 | opensea_slug: str 14 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: enhancement 6 | assignees: impreso 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Additional context** 17 | Add any other context or screenshots about the feature request here. 18 | -------------------------------------------------------------------------------- /open_rarity/models/utils/attribute_utils.py: -------------------------------------------------------------------------------- 1 | def normalize_attribute_string(value: str) -> str: 2 | """Normalizes either attribute names or string attribute values. 3 | This is a helper function to ensure we are consistently normalizing 4 | by always lower casing and stripping input string. 5 | 6 | Parameters 7 | ---------- 8 | value : str 9 | The string to normalize 10 | (this should be either attribute name or a string attribute value) 11 | 12 | Returns 13 | ------- 14 | str 15 | normalized string 16 | """ 17 | return value.lower().strip() 18 | -------------------------------------------------------------------------------- /open_rarity/models/token_standard.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class TokenStandard(Enum): 5 | """Enum class representing the interface or standard that 6 | a token is respecting. Each chain may have their own token standards. 7 | """ 8 | 9 | # -- Ethereum/EVM standards 10 | # https://eips.ethereum.org/EIPS/eip-721 11 | ERC721 = "erc721" 12 | # https://eips.ethereum.org/EIPS/eip-1155 13 | ERC1155 = "erc1155" 14 | 15 | # -- Solana token standards 16 | # https://docs.metaplex.com/programs/token-metadata/token-standard 17 | METAPLEX_NON_FUNGIBLE = "metaplex_non_fungible" 18 | -------------------------------------------------------------------------------- /icons/OpenRarity_Icons/03Black/Color 24x24-Black.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /icons/OpenRarity_Icons/04White/Color 24x24-White.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /icons/OpenRarity_Icons/03Black/Color 20x20-Black.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /icons/OpenRarity_Icons/04White/Color 20x20-White.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /icons/OpenRarity_Icons/03Black/Color 14x14-Black.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /icons/OpenRarity_Icons/03Black/Color 16x16-Black.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /icons/OpenRarity_Icons/04White/Color 14x14-White.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /icons/OpenRarity_Icons/04White/Color 16x16-White.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /.github/workflows/code-quality.yaml: -------------------------------------------------------------------------------- 1 | name: Code quality 2 | on: [push] 3 | jobs: 4 | code-quality: 5 | runs-on: ubuntu-latest 6 | 7 | steps: 8 | - uses: actions/checkout@v2 9 | 10 | - name: Install Poetry 11 | uses: snok/install-poetry@v1 12 | with: 13 | virtualenvs-create: true 14 | virtualenvs-in-project: true 15 | 16 | - name: Install dependencies 17 | run: poetry install --no-interaction --no-root 18 | 19 | - name: Linter 20 | run: poetry run flake8 21 | 22 | - name: Code Formatting 23 | run: poetry run black . 24 | 25 | - name: Type Check 26 | run: poetry run mypy open_rarity 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # Distribution / packaging 7 | .Python 8 | build/ 9 | develop-eggs/ 10 | dist/ 11 | downloads/ 12 | eggs/ 13 | .eggs/ 14 | lib/ 15 | lib64/ 16 | parts/ 17 | .python-version 18 | sdist/ 19 | var/ 20 | wheels/ 21 | share/python-wheels/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | MANIFEST 26 | 27 | # Code editors 28 | .vscode 29 | 30 | # Log files 31 | *.log 32 | 33 | # Test Coverage report 34 | .coverage 35 | 36 | # Output files from scripts 37 | testset*.csv 38 | score_real_collections_results*.json 39 | 40 | # Local cache files 41 | cached_data/*.json 42 | cached_data/*/*.json 43 | 44 | poetry.lock -------------------------------------------------------------------------------- /open_rarity/models/token_rarity.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | from open_rarity.models.token import Token 4 | from open_rarity.models.token_ranking_features import TokenRankingFeatures 5 | 6 | 7 | @dataclass 8 | class TokenRarity: 9 | """The class holds rarity and optional rank information along with the token 10 | 11 | Attributes 12 | ---------- 13 | score : float 14 | OpenRarity score for the token within collection 15 | token : Token 16 | token class 17 | token_features : TokenFeatures 18 | various token features 19 | rank: int | None 20 | rank of the token within the collection 21 | """ 22 | 23 | score: float 24 | token_features: TokenRankingFeatures 25 | token: Token 26 | rank: int | None = None 27 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: impreso 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **Information** 14 | 15 | * Collection link/name 16 | * Contract standard 17 | * Chain 18 | 19 | **To Reproduce** 20 | Steps to reproduce the behavior: 21 | 22 | 23 | **Expected behavior** 24 | A clear and concise description of what you expected to happen. 25 | 26 | **Screenshots** 27 | If applicable, add screenshots to help explain your problem. 28 | 29 | **Environment** 30 | - OS: [e.g. iOS] 31 | - Python [e.g. 3.10.3] 32 | - Library Version [e.g. 1.0] 33 | 34 | 35 | **Additional context** 36 | Add any other context about the problem here. 37 | -------------------------------------------------------------------------------- /tests/resolver/test_rarity_sniper.py: -------------------------------------------------------------------------------- 1 | from open_rarity.resolver.rarity_providers.rank_resolver import RankResolver 2 | from open_rarity.resolver.rarity_providers.rarity_sniper import RaritySniperResolver 3 | 4 | 5 | class TestRaritySniperResolver: 6 | BORED_APE_SLUG = "bored-ape-yacht-club" 7 | 8 | def test_get_rank(self): 9 | rank = RaritySniperResolver.get_rank( 10 | collection_slug=self.BORED_APE_SLUG, 11 | token_id=1020, 12 | ) 13 | # We don't check ranks since they can change ranks 14 | assert rank 15 | 16 | def test_get_rank_no_contract(self): 17 | rank = RaritySniperResolver.get_rank( 18 | collection_slug="non_existent_slug", token_id=1 19 | ) 20 | assert rank is None 21 | 22 | def test_rank_resolver_parent(self): 23 | assert isinstance(RaritySniperResolver, RankResolver) 24 | -------------------------------------------------------------------------------- /icons/OpenRarity_Icons/01Color_LightMode/Color 24x24-LightMode.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /icons/OpenRarity_Icons/02Color_DarkMode/Color 24x24-DarkMode.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /icons/OpenRarity_Icons/01Color_LightMode/Color 20x20-LightMode.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /icons/OpenRarity_Icons/02Color_DarkMode/Color 20x20-DarkMode.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /icons/OpenRarity_Icons/01Color_LightMode/Color 14x14-LightMode.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /icons/OpenRarity_Icons/01Color_LightMode/Color 16x16-LightMode.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /icons/OpenRarity_Icons/02Color_DarkMode/Color 14x14-DarkMode.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /icons/OpenRarity_Icons/02Color_DarkMode/Color 16x16-DarkMode.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributions guide 2 | 3 | OpenRarity is a cross-company effort to improve rarity computation for NFTs (Non-Fungible Tokens). The core collaboration group consists of four primary contributors: Curio, icy.tools, OpenSea and Proof 4 | 5 | OpenRarity is an open-source project with Apache 2.0 license and all contributions are welcome. Consider following steps when you request/propose contribution: 6 | 7 | * Have a question? Submit it on OpenRarity GitHub discussions page 8 | * Create GitHub issue/bug with description of the problem link 9 | * Submit Pull Request with proposed changes 10 | * To merge the change in the main branch you required to get at least 2 approvals from the project maintainer list 11 | * Always add a unit test with your changes. 12 | 13 | 14 | # Code formatting 15 | We use git-precommit hooks in OpenRarity repo. Install it with the following command 16 | 17 | `poetry run pre-commit install` 18 | 19 | 20 | # IDE 21 | We recommend to use VS Ccode with python support enabled. 22 | -------------------------------------------------------------------------------- /open_rarity/resolver/models/token_with_rarity_data.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from enum import Enum 3 | 4 | from open_rarity.models.token import Token 5 | 6 | 7 | class RankProvider(Enum): 8 | # external ranking providers 9 | TRAITS_SNIPER = "traits_sniper" 10 | RARITY_SNIFFER = "rarity_sniffer" 11 | RARITY_SNIPER = "rarity_sniper" 12 | 13 | # open rarity scoring 14 | OR_ARITHMETIC = "or_arithmetic" 15 | OR_GEOMETRIC = "or_geometric" 16 | OR_HARMONIC = "or_harmonic" 17 | OR_SUM = "or_sum" 18 | OR_INFORMATION_CONTENT = "or_information_content" 19 | 20 | 21 | EXTERNAL_RANK_PROVIDERS = [ 22 | RankProvider.TRAITS_SNIPER, 23 | RankProvider.RARITY_SNIFFER, 24 | RankProvider.RARITY_SNIPER, 25 | ] 26 | 27 | Rank = int 28 | Score = float 29 | 30 | 31 | @dataclass 32 | class RarityData: 33 | provider: RankProvider 34 | rank: Rank 35 | score: Score | None = None 36 | 37 | 38 | @dataclass 39 | class TokenWithRarityData: 40 | token: Token 41 | rarities: list[RarityData] 42 | -------------------------------------------------------------------------------- /tests/resolver/test_rarity_sniffer.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from open_rarity.resolver.rarity_providers.rank_resolver import RankResolver 4 | from open_rarity.resolver.rarity_providers.rarity_sniffer import RaritySnifferResolver 5 | 6 | 7 | class TestRaritySnifferResolver: 8 | BORED_APE_COLLECTION_ADDRESS = "0xbc4ca0eda7647a8ab7c2061c2e118a18a936f13d" 9 | 10 | @pytest.mark.skipif( 11 | "not config.getoption('--run-resolvers')", 12 | reason="This just verifies external APIs but should not block main library", 13 | ) 14 | def test_get_all_ranks(self): 15 | token_id_to_ranks = RaritySnifferResolver.get_all_ranks( 16 | contract_address=self.BORED_APE_COLLECTION_ADDRESS 17 | ) 18 | assert len(token_id_to_ranks) == 10_000 19 | 20 | @pytest.mark.skip(reason="raritysniffer.com domain is down") 21 | def test_get_all_ranks_no_contract(self): 22 | token_id_to_ranks = RaritySnifferResolver.get_all_ranks( 23 | contract_address="0x123" 24 | ) 25 | assert len(token_id_to_ranks) == 0 26 | 27 | def test_rank_resolver_parent(self): 28 | assert isinstance(RaritySnifferResolver, RankResolver) 29 | -------------------------------------------------------------------------------- /tests/models/test_token_identifier.py: -------------------------------------------------------------------------------- 1 | from open_rarity.models.token_identifier import ( 2 | EVMContractTokenIdentifier, 3 | SolanaMintAddressTokenIdentifier, 4 | ) 5 | 6 | 7 | class TestTokenIdentifier: 8 | def test_evm_token_identifier_hashable(self): 9 | address = "evm_address" 10 | tid_1 = EVMContractTokenIdentifier(contract_address=address, token_id=1) 11 | tid_2 = EVMContractTokenIdentifier(contract_address=address, token_id=2) 12 | dup_tid_1 = EVMContractTokenIdentifier(contract_address=address, token_id=1) 13 | assert {tid_1, tid_2, dup_tid_1} == {tid_1, tid_2} 14 | 15 | tid_dict = {tid_1: "value1", tid_2: "value2"} 16 | tid_dict.update({dup_tid_1: "updated_value1"}) 17 | assert tid_dict == {tid_1: "updated_value1", tid_2: "value2"} 18 | 19 | def test_solana_token_identifier_hashable(self): 20 | tid_1 = SolanaMintAddressTokenIdentifier(mint_address="address1") 21 | tid_2 = SolanaMintAddressTokenIdentifier(mint_address="address2") 22 | dup_tid_1 = SolanaMintAddressTokenIdentifier(mint_address="address1") 23 | assert {tid_1, tid_2, dup_tid_1} == {tid_1, tid_2} 24 | 25 | tid_dict = {tid_1: "value1", tid_2: "value2"} 26 | tid_dict.update({dup_tid_1: "updated_value1"}) 27 | assert tid_dict == {tid_1: "updated_value1", tid_2: "value2"} 28 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | # Standard hooks 3 | - repo: https://github.com/pre-commit/pre-commit-hooks 4 | rev: v3.4.0 5 | hooks: 6 | - id: check-case-conflict 7 | - id: check-merge-conflict 8 | - id: check-symlinks 9 | - id: check-yaml 10 | - id: debug-statements 11 | - id: end-of-file-fixer 12 | - id: mixed-line-ending 13 | - id: trailing-whitespace 14 | - repo: https://github.com/pycqa/isort 15 | rev: 5.10.1 16 | hooks: 17 | - id: isort 18 | name: isort (python) 19 | additional_dependencies: [toml] 20 | - id: isort 21 | name: isort (cython) 22 | types: [cython] 23 | - id: isort 24 | name: isort (pyi) 25 | types: [pyi] 26 | # Black 27 | - repo: https://github.com/psf/black 28 | rev: 22.3.0 29 | hooks: 30 | - id: black 31 | types: [python] 32 | # Flake8 33 | - repo: https://gitlab.com/pycqa/flake8 34 | rev: 4.0.1 35 | hooks: 36 | - id: flake8 37 | additional_dependencies: [flake8-bugbear, pep8-naming] 38 | 39 | # Mypy 40 | # run locally to ensure 41 | - repo: local 42 | hooks: 43 | - id: mypy 44 | name: mypy 45 | types: [python] 46 | entry: poetry run mypy open_rarity 47 | language: system 48 | always_run: true 49 | pass_filenames: false 50 | -------------------------------------------------------------------------------- /open_rarity/scoring/token_feature_extractor.py: -------------------------------------------------------------------------------- 1 | from open_rarity.models import Token 2 | from open_rarity.models.collection import Collection 3 | from open_rarity.models.token_ranking_features import TokenRankingFeatures 4 | 5 | 6 | class TokenFeatureExtractor: 7 | """ 8 | Utility class that extract features from tokens 9 | """ 10 | 11 | @staticmethod 12 | def extract_unique_attribute_count( 13 | token: Token, collection: Collection 14 | ) -> TokenRankingFeatures: 15 | """This method extracts unique attributes count from the token 16 | 17 | Parameters 18 | ---------- 19 | token : Token 20 | The token to extract features from 21 | collection : Collection 22 | The collection with the attributes frequency counts to base the 23 | token trait probabilities on to calculate score. 24 | 25 | Returns 26 | ------- 27 | TokenFeatures 28 | Token features wrapper class that contains extracted features 29 | 30 | """ 31 | 32 | unique_attributes_count: int = 0 33 | 34 | for string_attribute in token.metadata.string_attributes.values(): 35 | count = collection.total_tokens_with_attribute(string_attribute) 36 | 37 | if count == 1: 38 | unique_attributes_count += 1 39 | 40 | return TokenRankingFeatures(unique_attribute_count=unique_attributes_count) 41 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | authors = ["Dan Meshkov ", "Vicky Gong "] 3 | description = "Open-Rarity library is an open standard that provides an easy, explanable and reproducible computation for NFT rarity" 4 | license = "Apache-2.0" 5 | name = "open-rarity" 6 | version = "0.7.5" 7 | 8 | readme = "README.md" 9 | 10 | classifiers = [ 11 | "Programming Language :: Python :: 3", 12 | "Programming Language :: Python", 13 | "Intended Audience :: Developers", 14 | "License :: OSI Approved :: Apache Software License", 15 | "Natural Language :: English", 16 | "Typing :: Typed", 17 | "Topic :: Software Development :: Libraries", 18 | "Development Status :: 4 - Beta", 19 | "Programming Language :: Python :: 3.10", 20 | "Programming Language :: Python :: 3.11", 21 | ] 22 | 23 | [tool.poetry.dependencies] 24 | numpy = ">=1.23.1" 25 | pydantic = ">=1.9.1,<2.3" 26 | python = ">=3.10,<3.13" 27 | requests = ">=2.28.1" 28 | 29 | [tool.poetry.group.dev.dependencies] 30 | black = "^23.7.0" 31 | flake8 = "^5.0.2" 32 | flake8-bugbear = "^22.7.1" 33 | isort = "^5.10.1" 34 | mypy = "^0.982" 35 | pep8-naming = "^0.13.1" 36 | pre-commit = "^2.19.0" 37 | pytest = "^7.1" 38 | pytest-cov = "^3.0" 39 | pytest-mock = "^3.10.0" 40 | types-requests = ">=2.28.6" 41 | 42 | [tool.black] 43 | line-length = 88 44 | 45 | [build-system] 46 | build-backend = "poetry.core.masonry.api" 47 | requires = ["poetry-core>=1.0.0"] 48 | 49 | [tool.isort] 50 | include_trailing_comma = true 51 | known_first_party = "open_rarity" 52 | multi_line_output = 3 53 | profile = "black" 54 | 55 | -------------------------------------------------------------------------------- /scripts/score_generated_collection.py: -------------------------------------------------------------------------------- 1 | from open_rarity import Collection, OpenRarityScorer, Token 2 | from open_rarity.rarity_ranker import RarityRanker 3 | 4 | if __name__ == "__main__": 5 | scorer = OpenRarityScorer() 6 | 7 | collection = Collection( 8 | name="My Collection Name", 9 | tokens=[ 10 | Token.from_erc721( 11 | contract_address="0xa3049...", 12 | token_id=1, 13 | metadata_dict={"hat": "cap", "shirt": "blue"}, 14 | ), 15 | Token.from_erc721( 16 | contract_address="0xa3049...", 17 | token_id=2, 18 | metadata_dict={"hat": "visor", "shirt": "green"}, 19 | ), 20 | Token.from_erc721( 21 | contract_address="0xa3049...", 22 | token_id=3, 23 | metadata_dict={"hat": "visor", "shirt": "blue"}, 24 | ), 25 | ], 26 | ) # Replace inputs with your collection-specific details here 27 | 28 | # Generate scores for a collection 29 | token_scores = scorer.score_collection(collection=collection) 30 | 31 | print(f"Token scores for collection: {token_scores}") 32 | 33 | # Generate score for a single token in a collection 34 | token = collection.tokens[0] # Your token details filled in 35 | token_score = scorer.score_token(collection=collection, token=token) 36 | 37 | # Better yet.. just use ranker directly! 38 | ranked_tokens = RarityRanker.rank_collection(collection=collection) 39 | for ranked_token in ranked_tokens: 40 | print( 41 | f"Token {ranked_token.token} has rank {ranked_token.rank} " 42 | "and score {ranked_token.score}" 43 | ) 44 | -------------------------------------------------------------------------------- /open_rarity/scoring/scoring_handler.py: -------------------------------------------------------------------------------- 1 | from abc import abstractmethod 2 | from typing import Protocol 3 | 4 | from open_rarity.models.collection import Collection 5 | from open_rarity.models.token import Token 6 | 7 | 8 | class ScoringHandler(Protocol): 9 | """ScoringHandler class is an interface for different scoring algorithms to 10 | implement. Sub-classes are responsibile to ensure the batch functions are 11 | efficient for their particular algorithm. 12 | """ 13 | 14 | @abstractmethod 15 | def score_token(self, collection: Collection, token: Token) -> float: 16 | """Scores an individual token based on the traits distribution across 17 | the whole collection. 18 | 19 | Parameters 20 | ---------- 21 | collection : Collection 22 | The collection with the attributes frequency counts to base the 23 | token trait probabilities on to calculate score. 24 | token : Token 25 | The token to score 26 | 27 | Returns 28 | ------- 29 | float 30 | The token score 31 | """ 32 | raise NotImplementedError 33 | 34 | @abstractmethod 35 | def score_tokens( 36 | self, 37 | collection: Collection, 38 | tokens: list[Token], 39 | ) -> list[float]: 40 | """Used if you only want to score a batch of tokens that belong to collection. 41 | This will typically be more efficient than calling score_token for each 42 | token in `tokens`. 43 | 44 | Parameters 45 | ---------- 46 | collection : Collection 47 | The collection to score from 48 | tokens : list[Token] 49 | a batch of tokens belonging to collection to be scored 50 | 51 | Returns 52 | ------- 53 | list[float] 54 | list of scores in order of `tokens` 55 | """ 56 | raise NotImplementedError 57 | -------------------------------------------------------------------------------- /open_rarity/resolver/rarity_providers/rarity_sniper.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import requests 4 | 5 | from .rank_resolver import RankResolver 6 | 7 | logger = logging.getLogger("open_rarity_logger") 8 | RARITY_SNIPER_API_URL = ( 9 | "https://api.raritysniper.com/public/collection/{slug}/id/{token_id}" 10 | ) 11 | USER_AGENT = { 12 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36" # noqa: E501 13 | } 14 | 15 | 16 | class RaritySniperResolver(RankResolver): 17 | @staticmethod 18 | def get_all_ranks(slug: str) -> dict[str, int]: 19 | raise NotImplementedError 20 | 21 | @staticmethod 22 | def get_slug(opensea_slug: str) -> str: 23 | # custom fixes to normalize slug name 24 | # used in rarity sniper 25 | slug = opensea_slug.replace("-nft", "") 26 | slug = slug.replace("-official", "") 27 | slug = slug.replace("beanzofficial", "beanz") 28 | slug = slug.replace("boredapeyachtclub", "bored-ape-yacht-club") 29 | slug = slug.replace("clonex", "clone-x") 30 | slug = slug.replace("invisiblefriends", "invisible-friends") 31 | slug = slug.replace("proof-", "") 32 | slug = slug.replace("pudgypenguins", "pudgy-penguins") 33 | slug = slug.replace("wtf", "") 34 | 35 | return slug 36 | 37 | @staticmethod 38 | def get_rank(collection_slug: str, token_id: int) -> int | None: 39 | url = RARITY_SNIPER_API_URL.format(slug=collection_slug, token_id=token_id) 40 | logger.debug("{url}".format(url=url)) 41 | response = requests.request("GET", url, headers=USER_AGENT) 42 | if response.status_code == 200: 43 | return response.json()["rank"] 44 | else: 45 | logger.debug( 46 | f"[RaritySniper] Failed to resolve Rarity Sniper rank for " 47 | f"{collection_slug} {token_id}. Received {response.status_code} for " 48 | f"{url}: {response.reason}. {response.json()}" 49 | ) 50 | return None 51 | -------------------------------------------------------------------------------- /tests/scoring/test_token_feature_extractor.py: -------------------------------------------------------------------------------- 1 | from open_rarity.scoring.token_feature_extractor import TokenFeatureExtractor 2 | from tests.helpers import generate_collection_with_token_traits 3 | 4 | 5 | class TestFeatureExtractor: 6 | def test_feature_extractor(self): 7 | collection = generate_collection_with_token_traits( 8 | [ 9 | {"bottom": "1", "hat": "1", "special": "true"}, 10 | {"bottom": "1", "hat": "1", "special": "false"}, 11 | {"bottom": "2", "hat": "2", "special": "false"}, 12 | {"bottom": "2", "hat": "2", "special": "false"}, 13 | {"bottom": "3", "hat": "2", "special": "false"}, 14 | {"bottom": "4", "hat": "3", "special": "false"}, 15 | ] 16 | ) 17 | 18 | assert ( 19 | TokenFeatureExtractor.extract_unique_attribute_count( 20 | token=collection.tokens[0], collection=collection 21 | ).unique_attribute_count 22 | == 1 23 | ) 24 | 25 | assert ( 26 | TokenFeatureExtractor.extract_unique_attribute_count( 27 | token=collection.tokens[1], collection=collection 28 | ).unique_attribute_count 29 | == 0 30 | ) 31 | 32 | assert ( 33 | TokenFeatureExtractor.extract_unique_attribute_count( 34 | token=collection.tokens[2], collection=collection 35 | ).unique_attribute_count 36 | == 0 37 | ) 38 | 39 | assert ( 40 | TokenFeatureExtractor.extract_unique_attribute_count( 41 | token=collection.tokens[3], collection=collection 42 | ).unique_attribute_count 43 | == 0 44 | ) 45 | 46 | assert ( 47 | TokenFeatureExtractor.extract_unique_attribute_count( 48 | token=collection.tokens[4], collection=collection 49 | ).unique_attribute_count 50 | == 1 51 | ) 52 | 53 | assert ( 54 | TokenFeatureExtractor.extract_unique_attribute_count( 55 | token=collection.tokens[5], collection=collection 56 | ).unique_attribute_count 57 | == 2 58 | ) 59 | -------------------------------------------------------------------------------- /open_rarity/models/token_identifier.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Annotated, Literal, Type, TypeAlias, Union 3 | 4 | from pydantic import Field 5 | 6 | 7 | @dataclass(frozen=True) 8 | class EVMContractTokenIdentifier: 9 | """This token is identified by the contract address and token ID number. 10 | 11 | This identifier is based off of the interface as defined by ERC721 and ERC1155, 12 | where unique tokens belong to the same contract but have their own numeral token id. 13 | """ 14 | 15 | contract_address: str 16 | token_id: int 17 | identifier_type: Literal["evm_contract"] = "evm_contract" 18 | 19 | def __str__(self): 20 | return f"Contract({self.contract_address}) #{self.token_id}" 21 | 22 | @classmethod 23 | def from_dict(cls, data_dict: dict): 24 | return cls( 25 | contract_address=data_dict["contract_address"], 26 | token_id=data_dict["token_id"], 27 | ) 28 | 29 | def to_dict(self) -> dict: 30 | return { 31 | "contract_address": self.contract_address, 32 | "token_id": self.token_id, 33 | } 34 | 35 | 36 | @dataclass(frozen=True) 37 | class SolanaMintAddressTokenIdentifier: 38 | """This token is identified by their solana account address. 39 | 40 | This identifier is based off of the interface defined by the Solana SPL token 41 | standard where every such token is declared by creating a mint account. 42 | """ 43 | 44 | mint_address: str 45 | identifier_type: Literal["solana_mint_address"] = "solana_mint_address" 46 | 47 | def __str__(self): 48 | return f"MintAddress({self.mint_address})" 49 | 50 | @classmethod 51 | def from_dict(cls, data_dict: dict): 52 | return cls( 53 | mint_address=data_dict["mint_address"], 54 | ) 55 | 56 | def to_dict(self) -> dict: 57 | return { 58 | "mint_address": self.mint_address, 59 | } 60 | 61 | 62 | # This is used to specifies how the collection is identified and the 63 | # logic used to group the NFTs together 64 | TokenIdentifier = Annotated[ 65 | (EVMContractTokenIdentifier | SolanaMintAddressTokenIdentifier), 66 | Field(discriminator="identifier_type"), 67 | ] 68 | 69 | TokenIdentifierClass: TypeAlias = Union[ 70 | Type[EVMContractTokenIdentifier], Type[SolanaMintAddressTokenIdentifier] 71 | ] 72 | 73 | 74 | def get_identifier_class_from_dict(data_dict: dict) -> TokenIdentifierClass: 75 | return ( 76 | EVMContractTokenIdentifier 77 | if "token_id" in data_dict 78 | else SolanaMintAddressTokenIdentifier 79 | ) 80 | -------------------------------------------------------------------------------- /tests/resolver/test_trait_sniper.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from open_rarity.resolver.rarity_providers.rank_resolver import RankResolver 4 | from open_rarity.resolver.rarity_providers.trait_sniper import TraitSniperResolver 5 | 6 | # NOTE: API_KEY is needed for these tests (TRAIT_SNIPER_API_KEY must be set) 7 | 8 | 9 | class TestTraitSniperResolver: 10 | BORED_APE_COLLECTION_ADDRESS = "0xbc4ca0eda7647a8ab7c2061c2e118a18a936f13d" 11 | 12 | @pytest.mark.skipif( 13 | "not config.getoption('--run-resolvers')", 14 | reason="This tests runs too long due to rate limits to have as part of CI/CD " 15 | "but should be run whenver someone changes resolvers. Also needs API key", 16 | ) 17 | def test_get_all_ranks(self): 18 | token_ranks = TraitSniperResolver.get_all_ranks( 19 | contract_address=self.BORED_APE_COLLECTION_ADDRESS 20 | ) 21 | assert len(token_ranks) == 10000 22 | 23 | @pytest.mark.skipif( 24 | "not config.getoption('--run-resolvers')", 25 | reason="This requires API key", 26 | ) 27 | def test_get_ranks_first_page(self): 28 | token_ranks = TraitSniperResolver.get_ranks( 29 | contract_address=self.BORED_APE_COLLECTION_ADDRESS, page=1 30 | ) 31 | assert len(token_ranks) == 200 32 | 33 | @pytest.mark.skipif( 34 | "not config.getoption('--run-resolvers')", 35 | reason="This requires API key", 36 | ) 37 | def test_get_ranks_max_page(self): 38 | token_ranks = TraitSniperResolver.get_ranks( 39 | contract_address=self.BORED_APE_COLLECTION_ADDRESS, page=50 40 | ) 41 | assert len(token_ranks) == 200 42 | 43 | @pytest.mark.skipif( 44 | "not config.getoption('--run-resolvers')", 45 | reason="This requires API key", 46 | ) 47 | def test_get_ranks_no_more_data(self): 48 | token_ranks = TraitSniperResolver.get_ranks( 49 | contract_address=self.BORED_APE_COLLECTION_ADDRESS, page=51 50 | ) 51 | assert len(token_ranks) == 0 52 | 53 | @pytest.mark.skipif( 54 | "not config.getoption('--run-resolvers')", 55 | reason="This requires API key", 56 | ) 57 | def test_get_ranks_no_contract(self): 58 | token_ranks = TraitSniperResolver.get_ranks(contract_address="0x123", page=1) 59 | assert len(token_ranks) == 0 60 | 61 | @pytest.mark.skipif( 62 | "not config.getoption('--run-resolvers')", 63 | reason="This requires API key", 64 | ) 65 | def test_get_rank(self): 66 | rank = TraitSniperResolver.get_rank( 67 | collection_slug="boredapeyachtclub", 68 | token_id=1000, 69 | ) 70 | assert rank 71 | 72 | def test_rank_resolver_parent(self): 73 | assert isinstance(TraitSniperResolver, RankResolver) 74 | -------------------------------------------------------------------------------- /open_rarity/resolver/rarity_providers/rarity_sniffer.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import requests 4 | 5 | from .rank_resolver import RankResolver 6 | 7 | logger = logging.getLogger("open_rarity_logger") 8 | RARITY_SNIFFER_API_URL = "https://raritysniffer.com/api/index.php" 9 | USER_AGENT = { 10 | "User-Agent": ( 11 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " 12 | "(KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36" # noqa: E501 13 | ) 14 | } 15 | 16 | 17 | class RaritySnifferResolver(RankResolver): 18 | @staticmethod 19 | def get_all_ranks(contract_address: str) -> dict[str, int]: 20 | """Fetches all available tokens and ranks 21 | for a given collection from rarity sniffer. 22 | Only usable for EVM tokens and collections for a single 23 | contract address. 24 | 25 | Parameters 26 | ---------- 27 | contract_address : The contract address of the collection 28 | 29 | Returns 30 | ------- 31 | dict[int, int]: Dictionary of token ID # to the rank. Empty if no data. 32 | 33 | Raises 34 | ------ 35 | Exception 36 | If call to the rarity sniffer failed the method throws exception 37 | """ 38 | querystring = { 39 | "query": "fetch", 40 | "collection": contract_address, 41 | "taskId": "any", 42 | "norm": "true", 43 | "partial": "false", 44 | "traitCount": "true", 45 | } 46 | 47 | response = requests.request( 48 | "GET", 49 | RARITY_SNIFFER_API_URL, 50 | params=querystring, 51 | headers=USER_AGENT, 52 | ) 53 | 54 | if response.status_code != 200: 55 | logger.debug( 56 | "[RaritySniffer] Failed to resolve Rarity Sniffer ranks for " 57 | f"{contract_address}. Received: {response.status_code}: " 58 | f"{response.reason} {response.json()}" 59 | ) 60 | response.raise_for_status() 61 | 62 | data = response.json().get("data", None) 63 | if "Not found" in response.json().get("error", "") or not data: 64 | logger.exception( 65 | f"[RaritySniffer] No data for {contract_address}. " 66 | f"Json response: {response.json()}", 67 | exc_info=True, 68 | ) 69 | return {} 70 | try: 71 | token_ids_to_ranks = { 72 | str(nft["id"]): int(nft["positionId"]) for nft in data 73 | } 74 | except Exception: 75 | logger.exception( 76 | f"[RaritySniffer] Data could not be parsed for {contract_address}. " 77 | f"Json response: {response.json()}", 78 | exc_info=True, 79 | ) 80 | return {} 81 | 82 | return token_ids_to_ranks 83 | -------------------------------------------------------------------------------- /open_rarity/scoring/handlers/sum_scoring_handler.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from open_rarity.models.collection import Collection, CollectionAttribute 4 | from open_rarity.models.token import Token 5 | from open_rarity.models.token_metadata import AttributeName 6 | from open_rarity.scoring.utils import get_token_attributes_scores_and_weights 7 | 8 | 9 | class SumScoringHandler: 10 | """sum of n trait probabilities""" 11 | 12 | def __init__(self, normalized: bool = True): 13 | """ 14 | Parameters 15 | ---------- 16 | normalized : bool, optional 17 | If true, individual traits will be normalized based on total number 18 | of possible values for an attribute name, by default True. 19 | """ 20 | self.normalized = normalized 21 | 22 | def score_token(self, collection: Collection, token: Token) -> float: 23 | return self._score_token(collection, token, self.normalized) 24 | 25 | def score_tokens( 26 | self, 27 | collection: Collection, 28 | tokens: list[Token], 29 | ) -> list[float]: 30 | # Memoize for performance 31 | collection_null_attributes = collection.extract_null_attributes() 32 | return [ 33 | self._score_token( 34 | collection, t, self.normalized, collection_null_attributes 35 | ) 36 | for t in tokens 37 | ] 38 | 39 | # Private methods 40 | def _score_token( 41 | self, 42 | collection: Collection, 43 | token: Token, 44 | normalized: bool = True, 45 | collection_null_attributes: dict[AttributeName, CollectionAttribute] = None, 46 | ) -> float: 47 | """Calculates the score of the token by taking the sum of the attribute 48 | scores with weights. 49 | 50 | Parameters 51 | ---------- 52 | collection : Collection 53 | The collection with the attributes frequency counts to base the 54 | token trait probabilities on to calculate score. 55 | token : Token 56 | The token to score 57 | normalized : bool, optional 58 | Set to true to enable individual trait normalizations based on 59 | total number of possible values for an attribute name, by default True. 60 | collection_null_attributes : dict[AttributeName, CollectionAttribute], optional 61 | Optional memoization of collection.extract_null_attributes(), 62 | by default None. 63 | 64 | Returns 65 | ------- 66 | float 67 | The token score 68 | """ 69 | attr_scores, attr_weights = get_token_attributes_scores_and_weights( 70 | collection=collection, 71 | token=token, 72 | normalized=normalized, 73 | collection_null_attributes=collection_null_attributes, 74 | ) 75 | 76 | return np.dot(attr_scores, attr_weights) 77 | -------------------------------------------------------------------------------- /open_rarity/scoring/handlers/arithmetic_mean_scoring_handler.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from open_rarity.models.collection import Collection, CollectionAttribute 4 | from open_rarity.models.token import Token 5 | from open_rarity.models.token_metadata import AttributeName 6 | from open_rarity.scoring.utils import get_token_attributes_scores_and_weights 7 | 8 | 9 | class ArithmeticMeanScoringHandler: 10 | """arithmetic mean of a token's n trait probabilities""" 11 | 12 | def __init__(self, normalized: bool = True): 13 | """ 14 | Parameters 15 | ---------- 16 | normalized : bool, optional 17 | If true, individual traits will be normalized based on total number 18 | of possible values for an attribute name, by default True. 19 | """ 20 | self.normalized = normalized 21 | 22 | def score_token(self, collection: Collection, token: Token) -> float: 23 | return self._score_token(collection, token, self.normalized) 24 | 25 | def score_tokens( 26 | self, 27 | collection: Collection, 28 | tokens: list[Token], 29 | ) -> list[float]: 30 | collection_null_attributes = collection.extract_null_attributes() 31 | return [ 32 | self._score_token( 33 | collection, t, self.normalized, collection_null_attributes 34 | ) 35 | for t in tokens 36 | ] 37 | 38 | # Private methods 39 | def _score_token( 40 | self, 41 | collection: Collection, 42 | token: Token, 43 | normalized: bool = True, 44 | collection_null_attributes: dict[AttributeName, CollectionAttribute] = None, 45 | ) -> float: 46 | """Calculates the score of the token by taking the arithmetic mean of 47 | the attribute scores with weights. 48 | 49 | Parameters 50 | ---------- 51 | collection : Collection 52 | The collection with the attributes frequency counts to base the token 53 | trait probabilities on. 54 | token : Token 55 | The token to score. 56 | normalized : bool, optional 57 | Set to true to enable individual trait normalizations based on total 58 | number of possible values for an attribute, by default True. 59 | collection_null_attributes : dict[AttributeName, CollectionAttribute], optional 60 | Optional memoization of collection.extract_null_attributes(), by 61 | default None 62 | 63 | Returns 64 | ------- 65 | float 66 | The token score 67 | """ 68 | attr_scores, attr_weights = get_token_attributes_scores_and_weights( 69 | collection=collection, 70 | token=token, 71 | normalized=normalized, 72 | collection_null_attributes=collection_null_attributes, 73 | ) 74 | 75 | avg = float(np.average(attr_scores, weights=attr_weights)) 76 | return avg 77 | -------------------------------------------------------------------------------- /open_rarity/scoring/handlers/harmonic_mean_scoring_handler.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from open_rarity.models.collection import Collection, CollectionAttribute 4 | from open_rarity.models.token import Token 5 | from open_rarity.models.token_metadata import AttributeName 6 | from open_rarity.scoring.utils import get_token_attributes_scores_and_weights 7 | 8 | 9 | class HarmonicMeanScoringHandler: 10 | """harmonic mean of a token's n trait probabilities""" 11 | 12 | def __init__(self, normalized: bool = True): 13 | """ 14 | Parameters 15 | ---------- 16 | normalized : bool, optional 17 | If true, individual traits will be normalized based on total number 18 | of possible values for an attribute name, by default True. 19 | """ 20 | self.normalized = normalized 21 | 22 | def score_token(self, collection: Collection, token: Token) -> float: 23 | return self._score_token(collection, token, self.normalized) 24 | 25 | def score_tokens( 26 | self, 27 | collection: Collection, 28 | tokens: list[Token], 29 | ) -> list[float]: 30 | # Memoize for performance 31 | collection_null_attributes = collection.extract_null_attributes() 32 | return [ 33 | self._score_token( 34 | collection, t, self.normalized, collection_null_attributes 35 | ) 36 | for t in tokens 37 | ] 38 | 39 | # Private methods 40 | def _score_token( 41 | self, 42 | collection: Collection, 43 | token: Token, 44 | normalized: bool = True, 45 | collection_null_attributes: dict[AttributeName, CollectionAttribute] = None, 46 | ) -> float: 47 | """Calculates the score of the token by taking the harmonic mean of the 48 | attribute scores with weights. 49 | 50 | Parameters 51 | ---------- 52 | collection : Collection 53 | The collection with the attributes frequency counts to base the 54 | token trait probabilities on to calculate score. 55 | token : Token 56 | The token to score 57 | normalized : bool, optional 58 | Set to true to enable individual trait normalizations based on 59 | total number of possible values for an attribute name, by default True. 60 | collection_null_attributes : dict[AttributeName, CollectionAttribute], optional 61 | Optional memoization of collection.extract_null_attributes(), 62 | by default None. 63 | 64 | Returns 65 | ------- 66 | float 67 | The token score 68 | """ 69 | 70 | attr_scores, attr_weights = get_token_attributes_scores_and_weights( 71 | collection=collection, 72 | token=token, 73 | normalized=normalized, 74 | collection_null_attributes=collection_null_attributes, 75 | ) 76 | 77 | return float(np.average(np.reciprocal(attr_scores), weights=attr_weights) ** -1) 78 | -------------------------------------------------------------------------------- /open_rarity/scoring/handlers/geometric_mean_scoring_handler.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from open_rarity.models.collection import Collection, CollectionAttribute 4 | from open_rarity.models.token import Token 5 | from open_rarity.models.token_metadata import AttributeName 6 | from open_rarity.scoring.utils import get_token_attributes_scores_and_weights 7 | 8 | 9 | class GeometricMeanScoringHandler: 10 | """geometric mean of a token's n trait probabilities 11 | - equivalent to the nth root of the product of the trait probabilities 12 | - equivalent to the nth power of "statistical rarity" 13 | """ 14 | 15 | def __init__(self, normalized: bool = True): 16 | """ 17 | Parameters 18 | ---------- 19 | normalized : bool, optional 20 | If true, individual traits will be normalized based on total number 21 | of possible values for an attribute name, by default True. 22 | """ 23 | self.normalized = normalized 24 | 25 | def score_token(self, collection: Collection, token: Token) -> float: 26 | return self._score_token(collection, token, self.normalized) 27 | 28 | def score_tokens( 29 | self, 30 | collection: Collection, 31 | tokens: list[Token], 32 | ) -> list[float]: 33 | collection_null_attributes = collection.extract_null_attributes() 34 | return [ 35 | self._score_token( 36 | collection, t, self.normalized, collection_null_attributes 37 | ) 38 | for t in tokens 39 | ] 40 | 41 | # Private methods 42 | def _score_token( 43 | self, 44 | collection: Collection, 45 | token: Token, 46 | normalized: bool = True, 47 | collection_null_attributes: dict[AttributeName, CollectionAttribute] = None, 48 | ) -> float: 49 | """Calculates the score of the token by taking the geometric mean of the 50 | attribute scores with weights. 51 | 52 | Parameters 53 | ---------- 54 | collection : Collection 55 | The collection with the attributes frequency counts to base the 56 | token trait probabilities on to calculate score. 57 | token : Token 58 | The token to score 59 | normalized : bool, optional 60 | Set to true to enable individual trait normalizations based on 61 | total number of possible values for an attribute name, by default True. 62 | collection_null_attributes : dict[AttributeName, CollectionAttribute], optional 63 | Optional memoization of collection.extract_null_attributes(), 64 | by default None. 65 | 66 | Returns 67 | ------- 68 | float 69 | The token score 70 | """ 71 | attr_scores, attr_weights = get_token_attributes_scores_and_weights( 72 | collection=collection, 73 | token=token, 74 | normalized=normalized, 75 | collection_null_attributes=collection_null_attributes, 76 | ) 77 | 78 | return g_mean(attr_scores, weights=attr_weights) 79 | 80 | 81 | def g_mean(x, weights): 82 | a = np.log(x) 83 | return np.exp(np.average(a, axis=0, weights=weights)) 84 | -------------------------------------------------------------------------------- /tests/scoring/test_scorer.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from open_rarity import Collection, OpenRarityScorer, TokenStandard 4 | from open_rarity.scoring.handlers.information_content_scoring_handler import ( 5 | InformationContentScoringHandler, 6 | ) 7 | from tests.helpers import create_evm_token, generate_collection_with_token_traits 8 | 9 | 10 | class TestScorer: 11 | scorer = OpenRarityScorer() 12 | 13 | def test_init_settings(self): 14 | assert isinstance(self.scorer.handler, InformationContentScoringHandler) 15 | 16 | def test_score_collections_with_string_attributes(self): 17 | collection = generate_collection_with_token_traits( 18 | [ 19 | {"bottom": "1", "hat": "1", "special": "true"}, 20 | {"bottom": "1", "hat": "1", "special": "false"}, 21 | {"bottom": "2", "hat": "2", "special": "false"}, 22 | {"bottom": "2", "hat": "2", "special": "false"}, 23 | {"bottom": "pants", "hat": "cap", "special": "false"}, 24 | ] 25 | ) 26 | 27 | collection_two = generate_collection_with_token_traits( 28 | [ 29 | {"bottom": "1", "hat": "1", "special": "true"}, 30 | {"bottom": "1", "hat": "1", "special": "false"}, 31 | {"bottom": "2", "hat": "2", "special": "false"}, 32 | {"bottom": "2", "hat": "2", "special": "false"}, 33 | {"bottom": "pants", "hat": "cap", "special": "false"}, 34 | ] 35 | ) 36 | 37 | scores = self.scorer.score_collection(collection) 38 | assert len(scores) == 5 39 | 40 | scores = self.scorer.score_tokens(collection, collection.tokens) 41 | assert len(scores) == 5 42 | 43 | scores = self.scorer.score_collections([collection, collection_two]) 44 | assert len(scores[0]) == 5 45 | assert len(scores[1]) == 5 46 | 47 | def test_score_collection_with_numeric_attribute_errors(self): 48 | with pytest.raises(ValueError) as excinfo: 49 | collection = generate_collection_with_token_traits( 50 | [ 51 | {"bottom": "1", "hat": "1", "special": "true"}, 52 | {"bottom": "1", "hat": "1", "special": "false"}, 53 | {"bottom": "2", "hat": "2", "special": "false"}, 54 | {"bottom": "2", "hat": "2", "special": "false"}, 55 | {"bottom": 3, "hat": 2, "special": "false"}, 56 | ] 57 | ) 58 | 59 | scorer = OpenRarityScorer() 60 | scorer.score_collection(collection) 61 | 62 | assert ( 63 | "OpenRarity currently does not support collections " 64 | "with numeric or date traits" in str(excinfo.value) 65 | ) 66 | 67 | def test_score_collection_with_erc1155_errors(self): 68 | with pytest.raises(ValueError) as excinfo: 69 | collection = Collection( 70 | name="test", 71 | tokens=[ 72 | create_evm_token(token_id=i, token_standard=TokenStandard.ERC1155) 73 | for i in range(10) 74 | ], 75 | attributes_frequency_counts={}, 76 | ) 77 | scorer = OpenRarityScorer() 78 | scorer.score_collection(collection) 79 | 80 | assert ( 81 | "OpenRarity currently only supports ERC721/Non-fungible standards" 82 | in str(excinfo.value) 83 | ) 84 | -------------------------------------------------------------------------------- /open_rarity/scoring/utils.py: -------------------------------------------------------------------------------- 1 | from open_rarity.models.collection import Collection, CollectionAttribute 2 | from open_rarity.models.token import Token 3 | from open_rarity.models.token_metadata import AttributeName 4 | 5 | 6 | def get_token_attributes_scores_and_weights( 7 | collection: Collection, 8 | token: Token, 9 | normalized: bool, 10 | collection_null_attributes: dict[AttributeName, CollectionAttribute] = None, 11 | ) -> tuple[list[float], list[float]]: 12 | """Calculates the scores and normalization weights for a token 13 | based on its attributes. If the token does not have an attribute, the probability 14 | of the attribute being null is used instead. 15 | 16 | Parameters 17 | ---------- 18 | collection : Collection 19 | The collection to calculate probability on. 20 | token : Token 21 | The token to score. 22 | normalized : bool 23 | Set to true to enable individual trait normalizations based on total 24 | number of possible values for an attribute, by default True. 25 | collection_null_attributes : dict[ AttributeName, CollectionAttribute ], optional 26 | Optional memoization of collection.extract_null_attributes(), by default None. 27 | 28 | Returns 29 | ------- 30 | tuple[list[float], list[float]] 31 | A tuple of attribute scores and attribute weights. 32 | attribute scores: scores for an attribute is defined to be the inverse of 33 | the probability of that attribute existing across the collection. e.g. 34 | (total token supply / total tokens with that attribute name and value) 35 | attribute weights: The weights for each score that should be applied 36 | if normalization is to occur. 37 | """ 38 | # Create a combined attributes dictionary such that if the token has the attribute, 39 | # it uses the value's probability, and if it doesn't have the attribute, 40 | # uses the probability of that attribute being null. 41 | if collection_null_attributes is None: 42 | null_attributes = collection.extract_null_attributes() 43 | else: 44 | null_attributes = collection_null_attributes 45 | 46 | combined_attributes: dict[ 47 | str, CollectionAttribute 48 | ] = null_attributes | _convert_to_collection_attributes_dict(collection, token) 49 | 50 | sorted_attr_names = sorted(list(combined_attributes.keys())) 51 | sorted_attrs = [combined_attributes[attr_name] for attr_name in sorted_attr_names] 52 | 53 | total_supply = collection.token_total_supply 54 | 55 | # Normalize traits by dividing by the total number of possible values for 56 | # that trait. The normalization factor takes into account the cardinality 57 | # values for particual traits, such that high cardinality traits aren't 58 | # over-indexed in rarity. 59 | # Example: If Asset has a trait "Hat" and it has possible values 60 | # {"Red","Yellow","Green"} the normalization factor will be 1/3 or 61 | # 0.33. If a trait has 10,000 options, than the normalization factor is 1/10,000. 62 | if normalized: 63 | attr_weights = [ 64 | 1 / collection.total_attribute_values(attr_name) 65 | for attr_name in sorted_attr_names 66 | ] 67 | else: 68 | attr_weights = [1.0] * len(sorted_attr_names) 69 | 70 | scores = [total_supply / attr.total_tokens for attr in sorted_attrs] 71 | 72 | return (scores, attr_weights) 73 | 74 | 75 | def _convert_to_collection_attributes_dict(collection, token): 76 | # NOTE: We currently only support string attributes 77 | return { 78 | attribute.name: CollectionAttribute( 79 | attribute=attribute, 80 | total_tokens=collection.total_tokens_with_attribute(attribute), 81 | ) 82 | for attribute in token.metadata.string_attributes.values() 83 | } 84 | -------------------------------------------------------------------------------- /tests/resolver/test_external_rarity_provider.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from open_rarity.models.collection import Collection 4 | from open_rarity.models.token import Token 5 | from open_rarity.resolver.models.collection_with_metadata import CollectionWithMetadata 6 | from open_rarity.resolver.models.token_with_rarity_data import ( 7 | EXTERNAL_RANK_PROVIDERS, 8 | RankProvider, 9 | TokenWithRarityData, 10 | ) 11 | from open_rarity.resolver.rarity_providers.external_rarity_provider import ( 12 | ExternalRarityProvider, 13 | ) 14 | from open_rarity.resolver.rarity_providers.rarity_sniffer import RaritySnifferResolver 15 | from open_rarity.resolver.rarity_providers.rarity_sniper import RaritySniperResolver 16 | from open_rarity.resolver.rarity_providers.trait_sniper import TraitSniperResolver 17 | 18 | 19 | class TestExternalRarityProvider: 20 | bayc_address = "0xbc4ca0eda7647a8ab7c2061c2e118a18a936f13d" 21 | bayc_collection = CollectionWithMetadata( 22 | collection=Collection(tokens=[]), 23 | contract_addresses=[bayc_address], 24 | token_total_supply=10_000, 25 | opensea_slug="boredapeyachtclub", 26 | ) 27 | bayc_token_1 = Token.from_erc721( 28 | contract_address=bayc_address, 29 | token_id=1, 30 | metadata_dict={ 31 | "mouth": "grin", 32 | "background": "orange", 33 | "eyes": "blue beams", 34 | "fur": "robot", 35 | "clothes": "vietnam jacket", 36 | }, 37 | ) 38 | 39 | @pytest.mark.skipif( 40 | "not config.getoption('--run-resolvers')", 41 | reason="Dependent on external API and takes too long due to rate limits" 42 | "Should only be used to verify external API changes and needs " 43 | "TRAIT_SNIPER_API_KEY key", 44 | ) 45 | def test_fetch_and_update_ranks_real_api_calls(self): 46 | provider = ExternalRarityProvider() 47 | tokens_with_rarity = [TokenWithRarityData(token=self.bayc_token_1, rarities=[])] 48 | provider.fetch_and_update_ranks( 49 | collection_with_metadata=self.bayc_collection, 50 | tokens_with_rarity=tokens_with_rarity, 51 | cache_external_ranks=False, 52 | ) 53 | rarities = tokens_with_rarity[0].rarities 54 | assert len(rarities) == 3 55 | 56 | def test_fetch_and_update_ranks_mocked_api(self, mocker): 57 | provider = ExternalRarityProvider() 58 | tokens_with_rarity = [TokenWithRarityData(token=self.bayc_token_1, rarities=[])] 59 | 60 | rarity_sniffer_rank = 3256 61 | rarity_sniper_rank = 3250 62 | trait_sniper_rank = 3000 63 | mocker.patch.object( 64 | RaritySnifferResolver, 65 | "get_all_ranks", 66 | return_value={"1": rarity_sniffer_rank}, 67 | ) 68 | mocker.patch.object( 69 | TraitSniperResolver, "get_all_ranks", return_value={"1": trait_sniper_rank} 70 | ) 71 | mocker.patch.object( 72 | RaritySniperResolver, "get_rank", return_value=rarity_sniper_rank 73 | ) 74 | 75 | provider.fetch_and_update_ranks( 76 | collection_with_metadata=self.bayc_collection, 77 | tokens_with_rarity=tokens_with_rarity, 78 | cache_external_ranks=False, 79 | ) 80 | rarities = tokens_with_rarity[0].rarities 81 | assert len(rarities) == 3 82 | for rarity in rarities: 83 | assert rarity.provider in EXTERNAL_RANK_PROVIDERS 84 | if rarity.provider == RankProvider.RARITY_SNIFFER: 85 | assert rarity.rank == rarity_sniffer_rank 86 | elif rarity.provider == RankProvider.RARITY_SNIPER: 87 | assert rarity.rank == rarity_sniper_rank 88 | elif rarity.provider == RankProvider.TRAITS_SNIPER: 89 | assert rarity.rank == trait_sniper_rank 90 | else: 91 | raise Exception("Unexpected provider") 92 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![OpenRarity](img/OR_Github_banner.jpg) 2 | 3 | [![Version][version-badge]][version-link] 4 | [![Test CI][ci-badge]][ci-link] 5 | [![License][license-badge]][license-link] 6 | 7 | 8 | # OpenRarity 9 | 10 | We’re excited to announce OpenRarity, a new rarity protocol we’re building for the NFT community. Our objective is to provide a transparent rarity calculation that is entirely open-source, objective, and reproducible. 11 | 12 | With the explosion of new collections, marketplaces and tooling in the NFT ecosystem, we realized that rarity ranks often differed across platforms which could lead to confusion for buyers, sellers and creators. We believe it’s important to find a way to provide a unified and consistent set of rarity rankings across all platforms to help build more trust and transparency in the industry. 13 | 14 | We are releasing the OpenRarity library in a Beta preview to crowdsource feedback from the community and incorporate it into the library evolution. 15 | 16 | See the full announcement in the [blog post](https://mirror.xyz/openrarity.eth/-R8ZA5KCMgqtsueySlruAhB77YBX6fSnS_dT-8clZPQ). 17 | 18 | # Developer documentation 19 | 20 | Read [developer documentation](https://openrarity.gitbook.io/developers/) on how to integrate with OpenRarity. 21 | 22 | # Setup and run tests locally 23 | 24 | ``` 25 | poetry install # install dependencies locally 26 | poetry run pytest # run tests 27 | ``` 28 | 29 | Some tests are skipped by default due to it being more integration/slow tests. 30 | To run resolver tests: 31 | ``` 32 | poetry run pytest -k test_testset_resolver --run-resolvers 33 | ``` 34 | 35 | # Library usage 36 | You can install open rarity as a [python package](https://pypi.org/project/open-rarity/) to use OpenRarity in your project: 37 | ``` 38 | pip install open-rarity 39 | ``` 40 | Please refer to the [scripts/](/scripts/) folder for an example of how to use the library. 41 | 42 | If you have downloaded the repo, you can use OpenRarity shell tool to generate json or csv outputs of OpenRarity scoring and ranks for any collections: 43 | ``` 44 | python -m scripts.score_real_collections boredapeyachtclub proof-moonbirds 45 | ``` 46 | Read [developer documentation](https://openrarity.gitbook.io/developers/) for advanced library usage 47 | 48 | 49 | 50 | # Contributions guide and governance 51 | 52 | OpenRarity is a community effort to improve rarity computation for NFTs (Non-Fungible Tokens). The core collaboration group consists of four primary contributors: [Curio](https://curio.tools), [icy.tools](https://icy.tools), [OpenSea](https://opensea.io) and [Proof](https://www.proof.xyz/) 53 | 54 | OpenRarity is an open-source project and all contributions are welcome. Consider following steps when you request/propose contribution: 55 | 56 | - Have a question? Submit it on OpenRarity GitHub [discussions](https://github.com/ProjectOpenSea/open-rarity/discussions) page 57 | - Create GitHub issue/bug with description of the problem [link](https://github.com/ProjectOpenSea/open-rarity/issues/new?assignees=impreso&labels=bug&template=bug_report.md&title=) 58 | - Submit Pull Request with proposed changes 59 | - To merge the change in the `main` branch you required to get at least 2 approvals from the project maintainer list 60 | - Always add a unit test with your changes 61 | 62 | We use git-precommit hooks in OpenRarity repo. Install it with the following command 63 | ``` 64 | poetry run pre-commit install 65 | ``` 66 | 67 | # Project Setup and Core technologies 68 | 69 | We used the following core technologies in OpenRarity: 70 | 71 | - Python ≥ 3.10.x 72 | - Poetry for dependency management 73 | - Numpy ≥1.23.1 74 | - PyTest for unit tests 75 | 76 | # License 77 | 78 | Apache 2.0 , OpenSea, ICY, Curio, PROOF 79 | 80 | 81 | 82 | [license-badge]: https://img.shields.io/github/license/ProjectOpenSea/open-rarity 83 | [license-link]: https://github.com/ProjectOpenSea/open-rarity/blob/main/LICENSE 84 | [ci-badge]: https://github.com/ProjectOpenSea/open-rarity/actions/workflows/tests.yaml/badge.svg 85 | [ci-link]: https://github.com/ProjectOpenSea/open-rarity/actions/workflows/tests.yaml 86 | [version-badge]: https://img.shields.io/github/v/release/ProjectOpenSea/open-rarity 87 | [version-link]: https://github.com/ProjectOpenSea/open-rarity/releases?display_name=tag 88 | -------------------------------------------------------------------------------- /open_rarity/data/test_collections.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "collection_name": "Cool Cats", 4 | "collection_slug": "cool-cats-nft" 5 | }, 6 | { 7 | "collection_name": "Azuki", 8 | "collection_slug": "azuki" 9 | }, 10 | { 11 | "collection_name": "Bored Ape Yacht Club", 12 | "collection_slug": "boredapeyachtclub" 13 | }, 14 | { 15 | "collection_name": "Moonbirds", 16 | "collection_slug": "proof-moonbirds" 17 | }, 18 | { 19 | "collection_name": "Invisible Friends", 20 | "collection_slug": "invisiblefriends" 21 | }, 22 | { 23 | "collection_name": "Mutant Ape Yacht Club", 24 | "collection_slug": "mutant-ape-yacht-club" 25 | }, 26 | { 27 | "collection_name": "Doodles", 28 | "collection_slug": "doodles-official" 29 | }, 30 | { 31 | "collection_name": "goblintown.wtf", 32 | "collection_slug": "goblintownwtf" 33 | }, 34 | { 35 | "collection_name": "Mfers", 36 | "collection_slug": "mfers" 37 | }, 38 | { 39 | "collection_name": "Meebits", 40 | "collection_slug": "meebits" 41 | }, 42 | { 43 | "collection_name": "World of Women", 44 | "collection_slug": "world-of-women-nft" 45 | }, 46 | { 47 | "collection_name": "Moonbirds", 48 | "collection_slug": "moonbirds-oddities" 49 | }, 50 | { 51 | "collection_name": "Clonex", 52 | "collection_slug": "clonex" 53 | }, 54 | { 55 | "collection_name": "Beanz", 56 | "collection_slug": "beanzofficial" 57 | }, 58 | { 59 | "collection_name": "PudgyPenguins", 60 | "collection_slug": "pudgypenguins" 61 | }, 62 | { 63 | "collection_name": "Fidenza", 64 | "collection_slug": "fidenza-by-tyler-hobbs" 65 | }, 66 | { 67 | "collection_name": "PXN: Ghost Division", 68 | "collection_slug": "pxnghostdivision" 69 | }, 70 | { 71 | "collection_name": "Karafuru", 72 | "collection_slug": "karafuru" 73 | }, 74 | { 75 | "collection_name": "Hashmasks", 76 | "collection_slug": "hashmasks" 77 | }, 78 | { 79 | "collection_name": "FLUF World", 80 | "collection_slug": "fluf" 81 | }, 82 | { 83 | "collection_name": "Creature World", 84 | "collection_slug": "creatureworld" 85 | }, 86 | { 87 | "collection_name": "Phanta Bear", 88 | "collection_slug": "creatureworld" 89 | }, 90 | { 91 | "collection_name": "3 Landers", 92 | "collection_slug": "3landers" 93 | }, 94 | { 95 | "collection_name": "Lazy Lions", 96 | "collection_slug": "lazy-lions" 97 | }, 98 | { 99 | "collection_name": "Genesis Creepz", 100 | "collection_slug": "genesis-creepz" 101 | }, 102 | { 103 | "collection_name": "Kawmi", 104 | "collection_slug": "kiwami-genesis" 105 | }, 106 | { 107 | "collection_name": "Degentoonz", 108 | "collection_slug": "degentoonz-collection" 109 | }, 110 | { 111 | "collection_name": "Doge Pound", 112 | "collection_slug": "the-doge-pound" 113 | }, 114 | { 115 | "collection_name": "Everai", 116 | "collection_slug": "everai-heroes-duo" 117 | }, 118 | { 119 | "collection_name": "Los Muertos World", 120 | "collection_slug": "los-muertos-world" 121 | }, 122 | { 123 | "collection_name": "Bonji", 124 | "collection_slug": "boonjiproject" 125 | }, 126 | { 127 | "collection_name": "fanggangnft", 128 | "collection_slug": "fanggangnft" 129 | }, 130 | { 131 | "collection_name": "deadheads", 132 | "collection_slug": "deadheads" 133 | }, 134 | { 135 | "collection_name": "wolf-game-farmer", 136 | "collection_slug": "wolf-game-farmer" 137 | }, 138 | { 139 | "collection_name": "yoloholiday", 140 | "collection_slug": "yoloholiday" 141 | }, 142 | { 143 | "collection_name": "the-weirdo-ghost-gang", 144 | "collection_slug": "the-weirdo-ghost-gang" 145 | }, 146 | { 147 | "collection_name": "woodies-generative", 148 | "collection_slug": "woodies-generative" 149 | }, 150 | { 151 | "collection_name": "nostalgia-key", 152 | "collection_slug": "nostalgia-key" 153 | }, 154 | { 155 | "collection_name": "y00ts-yacht-club", 156 | "collection_slug": "y00ts-yacht-club" 157 | }, 158 | { 159 | "collection_name": "50yearsofatari", 160 | "collection_slug": "50yearsofatari" 161 | }, 162 | { 163 | "collection_name": "ctomgkirby", 164 | "collection_slug": "ctomgkirby" 165 | }, 166 | { 167 | "collection_name": "proofofpepe", 168 | "collection_slug": "proofofpepe" 169 | }, 170 | { 171 | "collection_name": "fat-rat-mafia", 172 | "collection_slug": "fat-rat-mafia" 173 | } 174 | ] 175 | -------------------------------------------------------------------------------- /scripts/score_real_collections.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import csv 3 | import json 4 | 5 | from open_rarity import RarityRanker 6 | from open_rarity.models.token_rarity import TokenRarity 7 | from open_rarity.resolver.opensea_api_helpers import get_collection_from_opensea 8 | 9 | parser = argparse.ArgumentParser() 10 | parser.add_argument( 11 | "slugs", 12 | type=str, 13 | default=["boredapeyachtclub"], 14 | nargs="*", 15 | help="Slugs you want to generate scoring for, separated by spaces", 16 | ) 17 | parser.add_argument( 18 | "--filename_prefix", 19 | dest="filename_prefix", 20 | default="score_real_collections_results", 21 | help="The filename prefix to output the ranking results to. " 22 | "The filename will be {prefix}_{slug}.json/csv", 23 | ) 24 | 25 | parser.add_argument( 26 | "--filetype", 27 | dest="filetype", 28 | default="json", 29 | choices=["json", "csv"], 30 | help="Determines output file type. Either 'json' or 'csv'.", 31 | ) 32 | 33 | parser.add_argument( 34 | "--cache", 35 | action=argparse.BooleanOptionalAction, 36 | dest="use_cache", 37 | default=True, 38 | help="Determines whether we force refetch all Token and trait data " 39 | "from Opensea or read data from a local cache file", 40 | ) 41 | 42 | 43 | def score_collection_and_output_results( 44 | slug: str, output_filename: str, use_cache: bool 45 | ): 46 | # Get collection 47 | collection = get_collection_from_opensea(slug, use_cache=use_cache) 48 | print(f"Created collection {slug} with {collection.token_total_supply} tokens") 49 | 50 | # Score, rank and sort ascending by token rarity rank 51 | sorted_token_rarities: list[TokenRarity] = RarityRanker.rank_collection( 52 | collection=collection 53 | ) 54 | 55 | # Print out ranks and scores 56 | print("Token ID and their ranks and scores, sorted by rank") 57 | json_output = {} 58 | csv_rows = [] 59 | for rarity_token in sorted_token_rarities: 60 | token_id = rarity_token.token.token_identifier.token_id 61 | rank = rarity_token.rank 62 | score = rarity_token.score 63 | json_output[token_id] = {"rank": rank, "score": score} 64 | csv_rows.append([token_id, rank, score]) 65 | 66 | # Write to json 67 | if output_filename.endswith(".json"): 68 | with open(output_filename, "w") as jsonfile: 69 | json.dump(json_output, jsonfile, indent=4) 70 | 71 | # Write to csv 72 | if output_filename.endswith(".csv"): 73 | with open(output_filename, "w") as csvfile: 74 | writer = csv.writer(csvfile) 75 | # headers 76 | writer.writerow(["token_id", "rank", "score"]) 77 | # content 78 | writer.writerows(csv_rows) 79 | 80 | 81 | if __name__ == "__main__": 82 | """This script by default fetches bored ape yacht club collection and token 83 | metadata from the Opensea API via opensea_api_helpers and scores + ranks the 84 | collection via OpenRarity scorer. 85 | 86 | It will output results into {prefix}_{slug}.json/csv file, with the default 87 | filename being "score_real_collections_results.json". 88 | 89 | If JSON, format is: 90 | { 91 | : { 92 | "rank": , 93 | "score": , 94 | } 95 | } 96 | 97 | If CSV, format is: 98 | Columns: Token ID, Rank, Score 99 | 100 | Command: 101 | `python -m scripts.score_real_collections` 102 | 103 | If you want to set your own slugs to process, you may pass it in via 104 | command-line. 105 | Example: 106 | `python -m scripts.score_real_collections boredapeyachtclub proof-moonbirds` 107 | """ 108 | args = parser.parse_args() 109 | use_cache = args.use_cache 110 | print(f"Scoring collections: {args.slugs} with {use_cache=}") 111 | print(f"Output file prefix: {args.filename_prefix} with type .{args.filetype}") 112 | 113 | files = [] 114 | for slug in args.slugs: 115 | output_filename = f"{args.filename_prefix}_{slug}.{args.filetype}" 116 | print(f"Generating results for: {slug}") 117 | score_collection_and_output_results( 118 | slug=slug, 119 | output_filename=output_filename, 120 | use_cache=use_cache, 121 | ) 122 | print(f"Outputted results to: {output_filename}") 123 | files.append(output_filename) 124 | 125 | print("Finished scoring and ranking collections. Output files:") 126 | for file in files: 127 | print(f"\t{file}") 128 | -------------------------------------------------------------------------------- /open_rarity/scoring/scorer.py: -------------------------------------------------------------------------------- 1 | from open_rarity.models.collection import Collection 2 | from open_rarity.models.token import Token 3 | from open_rarity.models.token_standard import TokenStandard 4 | from open_rarity.scoring.handlers.information_content_scoring_handler import ( 5 | InformationContentScoringHandler, 6 | ) 7 | from open_rarity.scoring.scoring_handler import ScoringHandler 8 | 9 | 10 | class Scorer: 11 | """Scorer is the main class to score rarity scores for a given 12 | collection and token(s) based on the default OpenRarity scoring 13 | algorithm. 14 | """ 15 | 16 | handler: ScoringHandler 17 | 18 | def __init__(self) -> None: 19 | # OpenRarity uses InformationContent as the scoring algorithm of choice. 20 | self.handler = InformationContentScoringHandler() 21 | 22 | def validate_collection(self, collection: Collection) -> None: 23 | """Validate collection eligibility for OpenRarity scoring 24 | 25 | Parameters 26 | ---------- 27 | collection: Collection) 28 | The collection to validate 29 | """ 30 | if collection.has_numeric_attribute: 31 | raise ValueError( 32 | "OpenRarity currently does not support collections with " 33 | "numeric or date traits" 34 | ) 35 | 36 | allowed_standards = { 37 | TokenStandard.ERC721, 38 | TokenStandard.METAPLEX_NON_FUNGIBLE, 39 | } 40 | 41 | if not set(collection.token_standards).issubset(allowed_standards): 42 | raise ValueError( 43 | "OpenRarity currently only supports ERC721/Non-fungible standards" 44 | ) 45 | 46 | def score_token(self, collection: Collection, token: Token) -> float: 47 | """Scores an individual token based on the traits distribution across 48 | the whole collection. 49 | 50 | Parameters 51 | ---------- 52 | collection : Collection 53 | The collection to score from 54 | token : Token 55 | a single Token to score 56 | 57 | Returns 58 | ------- 59 | float 60 | The score of the token 61 | """ 62 | self.validate_collection(collection=collection) 63 | return self.handler.score_token(collection=collection, token=token) 64 | 65 | def score_tokens(self, collection: Collection, tokens: list[Token]) -> list[float]: 66 | """Used if you only want to score a batch of tokens that belong to collection. 67 | This will typically be more efficient than calling score_token for each 68 | token in `tokens`. 69 | 70 | Parameters 71 | ---------- 72 | collection : Collection 73 | The collection to score from 74 | tokens : list[Token] 75 | a batch of tokens belonging to collection to be scored 76 | 77 | Returns 78 | ------- 79 | list[float] 80 | list of scores in order of `tokens` 81 | """ 82 | self.validate_collection(collection=collection) 83 | return self.handler.score_tokens(collection=collection, tokens=tokens) 84 | 85 | def score_collection(self, collection: Collection) -> list[float]: 86 | """Scores all tokens on collection.tokens 87 | 88 | Parameters 89 | ---------- 90 | collection : Collection 91 | The collection to score all tokens from 92 | 93 | Returns 94 | ------- 95 | list[float] 96 | list of scores in order of `collection.tokens` 97 | """ 98 | self.validate_collection(collection=collection) 99 | return self.handler.score_tokens( 100 | collection=collection, 101 | tokens=collection.tokens, 102 | ) 103 | 104 | def score_collections(self, collections: list[Collection]) -> list[list[float]]: 105 | """Scores all tokens in every collection provided. 106 | 107 | Parameters 108 | ---------- 109 | collections: list[Collection]) 110 | The collections to score 111 | 112 | Returns 113 | ------- 114 | list[list[float]] 115 | A list of scores for all tokens in each given Collection, 116 | ordered by the collection's `tokens` field. 117 | """ 118 | for collection in collections: 119 | self.validate_collection(collection=collection) 120 | return [ 121 | self.handler.score_tokens(collection=c, tokens=c.tokens) 122 | for c in collections 123 | ] 124 | -------------------------------------------------------------------------------- /open_rarity/rarity_ranker.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | from open_rarity.models.collection import Collection 4 | from open_rarity.models.token_rarity import TokenRarity 5 | from open_rarity.scoring.scorer import Scorer 6 | from open_rarity.scoring.token_feature_extractor import TokenFeatureExtractor 7 | 8 | 9 | class RarityRanker: 10 | """This class is used to rank a set of tokens given their rarity scores.""" 11 | 12 | default_scorer = Scorer() 13 | 14 | @staticmethod 15 | def rank_collection( 16 | collection: Collection, scorer: Scorer = default_scorer 17 | ) -> list[TokenRarity]: 18 | """ 19 | Ranks tokens in the collection with the default scorer implementation. 20 | Scores that are higher indicate a higher rarity, and thus a lower rank. 21 | 22 | Tokens with the same score will be assigned the same rank, e.g. we use RANK 23 | (vs. DENSE_RANK). 24 | Example: 1, 2, 2, 2, 5. 25 | Scores are considered the same rank if they are within about 9 decimal digits 26 | of each other. 27 | 28 | 29 | Parameters 30 | ---------- 31 | collection : Collection 32 | Collection object with populated tokens 33 | scorer: Scorer 34 | Scorer instance 35 | 36 | Returns 37 | ------- 38 | list[TokenRarity] 39 | list of TokenRarity objects with score, rank and token information 40 | sorted by rank 41 | """ 42 | 43 | if ( 44 | collection is None 45 | or collection.tokens is None 46 | or len(collection.tokens) == 0 47 | ): 48 | return [] 49 | 50 | tokens = collection.tokens 51 | scores: list[float] = scorer.score_tokens(collection, tokens=tokens) 52 | 53 | # fail ranking if dimension of scores doesn't match dimension of tokens 54 | assert len(tokens) == len(scores) 55 | 56 | token_rarities: list[TokenRarity] = [] 57 | 58 | # augment collection tokens with score information 59 | for idx, token in enumerate(tokens): 60 | # extract features from the token 61 | token_features = TokenFeatureExtractor.extract_unique_attribute_count( 62 | token=token, collection=collection 63 | ) 64 | 65 | token_rarities.append( 66 | TokenRarity( 67 | token=token, 68 | score=scores[idx], 69 | token_features=token_features, 70 | ) 71 | ) 72 | 73 | return RarityRanker.set_rarity_ranks(token_rarities) 74 | 75 | @staticmethod 76 | def set_rarity_ranks( 77 | token_rarities: list[TokenRarity], 78 | ) -> list[TokenRarity]: 79 | """Ranks a set of tokens according to OpenRarity algorithm. 80 | To account for additional factors like unique items in a collection, 81 | OpenRarity implements multi-factor sort. Current sort algorithm uses two 82 | factors: unique attributes count and Information Content score, in order. 83 | Tokens with the same score will be assigned the same rank, e.g. we use RANK 84 | (vs. DENSE_RANK). 85 | Example: 1, 2, 2, 2, 5. 86 | Scores are considered the same rank if they are within about 9 decimal digits 87 | of each other. 88 | 89 | Parameters 90 | ---------- 91 | token_rarities : list[TokenRarity] 92 | unordered list of tokens with rarity score 93 | information that should have the ranks set on 94 | 95 | Returns 96 | ------- 97 | list[TokenRarity] 98 | modified input token_rarities with ranking data set, 99 | ordered by rank ascending and score descending 100 | 101 | """ 102 | sorted_token_rarities: list[TokenRarity] = sorted( 103 | token_rarities, 104 | key=lambda k: ( 105 | k.token_features.unique_attribute_count, 106 | k.score, 107 | ), 108 | reverse=True, 109 | ) 110 | 111 | # Perform ranking of each token in collection 112 | for i, token_rarity in enumerate(sorted_token_rarities): 113 | rank = i + 1 114 | if i > 0: 115 | prev_token_rarity = sorted_token_rarities[i - 1] 116 | scores_equal = math.isclose(token_rarity.score, prev_token_rarity.score) 117 | if scores_equal: 118 | assert prev_token_rarity.rank is not None 119 | rank = prev_token_rarity.rank 120 | 121 | token_rarity.rank = rank 122 | 123 | return sorted_token_rarities 124 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | openrarity@gmail.com. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /open_rarity/models/token.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Any, Iterable 3 | 4 | from open_rarity.models.token_identifier import ( 5 | EVMContractTokenIdentifier, 6 | SolanaMintAddressTokenIdentifier, 7 | TokenIdentifier, 8 | get_identifier_class_from_dict, 9 | ) 10 | from open_rarity.models.token_metadata import ( 11 | Attribute, 12 | AttributeName, 13 | StringAttribute, 14 | TokenMetadata, 15 | ) 16 | from open_rarity.models.token_standard import TokenStandard 17 | from open_rarity.models.utils.attribute_utils import normalize_attribute_string 18 | 19 | 20 | @dataclass 21 | class Token: 22 | """Class represents a token on the blockchain. 23 | Examples of these are non-fungible tokens, or semi-fungible tokens. 24 | 25 | Attributes 26 | ---------- 27 | token_identifier : TokenIdentifier 28 | data representing how the token is identified, which may be based 29 | on the token_standard or chain it lives on. 30 | token_standard : TokenStandard 31 | name of token standard (e.g. EIP-721 or EIP-1155) 32 | metadata: TokenMetadata 33 | contains the metadata of this specific token 34 | """ 35 | 36 | token_identifier: TokenIdentifier 37 | token_standard: TokenStandard 38 | metadata: TokenMetadata 39 | 40 | @classmethod 41 | def from_erc721( 42 | cls, 43 | contract_address: str, 44 | token_id: int, 45 | metadata_dict: dict[AttributeName, Any], 46 | ): 47 | """Creates a Token class representing an ERC721 evm token given the following 48 | parameters. 49 | 50 | Parameters 51 | ---------- 52 | contract_address : str 53 | Contract address of the token 54 | token_id : int 55 | Token ID number of the token 56 | metadata_dict : dict 57 | Dictionary of attribute name to attribute value for the given token. 58 | The type of the value determines whether the attribute is a string, 59 | numeric or date attribute. 60 | 61 | class attribute type 62 | ------------ ------------- 63 | string string attribute 64 | int | float numeric_attribute 65 | datetime date_attribute (stored as timestamp, seconds from epoch) 66 | 67 | Returns 68 | ------- 69 | Token 70 | A Token instance with EVMContractTokenIdentifier and ERC721 standard set. 71 | """ 72 | return cls( 73 | token_identifier=EVMContractTokenIdentifier( 74 | contract_address=contract_address, token_id=token_id 75 | ), 76 | token_standard=TokenStandard.ERC721, 77 | metadata=TokenMetadata.from_attributes(metadata_dict), 78 | ) 79 | 80 | @classmethod 81 | def from_metaplex_non_fungible( 82 | cls, mint_address: str, attributes: dict[AttributeName, Any] 83 | ): 84 | """Creates a Token class representing a Metaplex non-fungible token 85 | given the following parameters. 86 | 87 | Parameters 88 | ---------- 89 | mint_address: str 90 | The mint address of the token. 91 | attributes : dict 92 | Dictionary of attribute name to attribute value for the given token. 93 | Same as the attributes in from_erc721. 94 | 95 | Returns 96 | ------- 97 | Token 98 | A Token instance with SolanaMintAddressTokenIdentifier and 99 | METAPLEX_NON_FUNGIBLE standard set. 100 | """ 101 | return cls( 102 | token_identifier=SolanaMintAddressTokenIdentifier( 103 | mint_address=mint_address, 104 | ), 105 | token_standard=TokenStandard.METAPLEX_NON_FUNGIBLE, 106 | metadata=TokenMetadata.from_attributes(attributes), 107 | ) 108 | 109 | @classmethod 110 | def from_dict(cls, data_dict: dict): 111 | identifier_class = get_identifier_class_from_dict(data_dict["token_identifier"]) 112 | 113 | return cls( 114 | token_identifier=identifier_class.from_dict(data_dict["token_identifier"]), 115 | token_standard=TokenStandard[data_dict["token_standard"]], 116 | metadata=TokenMetadata.from_attributes(data_dict["metadata_dict"]), 117 | ) 118 | 119 | def attributes(self) -> dict[AttributeName, Any]: 120 | return self.metadata.to_attributes() 121 | 122 | def has_attribute(self, attribute_name: str) -> bool: 123 | return self.metadata.attribute_exists(attribute_name) 124 | 125 | def trait_count(self) -> int: 126 | """Returns the count of non-null, non-"none" value traits this token has.""" 127 | 128 | def get_attributes_count(attributes: Iterable[Attribute]) -> int: 129 | return sum( 130 | map( 131 | lambda a: ( 132 | not isinstance(a, StringAttribute) 133 | or normalize_attribute_string(a.value) not in ("none", "") 134 | ), 135 | attributes, 136 | ) 137 | ) 138 | 139 | return ( 140 | get_attributes_count(self.metadata.string_attributes.values()) 141 | + get_attributes_count(self.metadata.numeric_attributes.values()) 142 | + get_attributes_count(self.metadata.date_attributes.values()) 143 | ) 144 | 145 | def to_dict(self) -> dict: 146 | return { 147 | "token_identifier": self.token_identifier.to_dict(), 148 | "metadata_dict": self.attributes(), 149 | "token_standard": self.token_standard.name, 150 | } 151 | 152 | def __str__(self): 153 | return f"Token[{self.token_identifier}]" 154 | -------------------------------------------------------------------------------- /tests/models/test_token_metadata.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | import pytest 4 | 5 | from open_rarity.models.token_metadata import ( 6 | DateAttribute, 7 | NumericAttribute, 8 | StringAttribute, 9 | TokenMetadata, 10 | ) 11 | 12 | now = datetime.now() 13 | 14 | 15 | class TestTokenMetadata: 16 | token_metadata = TokenMetadata( 17 | string_attributes={ 18 | "hat": StringAttribute(name="hat", value="blue cap"), 19 | }, 20 | numeric_attributes={ 21 | "integer trait": NumericAttribute(name="integer trait", value=1), 22 | }, 23 | date_attributes={ 24 | "created": DateAttribute(name="created", value=int(now.timestamp())), 25 | }, 26 | ) 27 | 28 | def test_from_attributes_valid_types(self): 29 | created = datetime.now() 30 | token_metadata = TokenMetadata.from_attributes( 31 | { 32 | "hat": "blue cap", 33 | "created": created, 34 | "integer trait": 1, 35 | "float trait": 203.5, 36 | "PANTS ": "jeans", 37 | } 38 | ) 39 | 40 | assert token_metadata.string_attributes == { 41 | "hat": StringAttribute(name="hat", value="blue cap"), 42 | "pants": StringAttribute(name="pants", value="jeans"), 43 | } 44 | assert token_metadata.numeric_attributes == { 45 | "integer trait": NumericAttribute(name="integer trait", value=1), 46 | "float trait": NumericAttribute(name="float trait", value=203.5), 47 | } 48 | assert token_metadata.date_attributes == { 49 | "created": DateAttribute(name="created", value=int(created.timestamp())), 50 | } 51 | 52 | def test_from_attributes_invalid_type(self): 53 | with pytest.raises(TypeError) as excinfo: 54 | TokenMetadata.from_attributes( 55 | { 56 | "hat": "blue cap", 57 | "created": {"bad input": "true"}, 58 | "integer trait": 1, 59 | "float trait": 203.5, 60 | } 61 | ) 62 | 63 | assert "Provided attribute value has invalid type" in str(excinfo.value) 64 | 65 | def test_attribute_exists(self): 66 | assert self.token_metadata.attribute_exists("hat") 67 | assert self.token_metadata.attribute_exists("HAT") 68 | assert self.token_metadata.attribute_exists("integer trait") 69 | assert self.token_metadata.attribute_exists("integer trait ") 70 | assert self.token_metadata.attribute_exists("created") 71 | assert not self.token_metadata.attribute_exists("scarf") 72 | assert not TokenMetadata().attribute_exists("hat") 73 | 74 | def test_add_attribute_empty(self): 75 | token_metadata = TokenMetadata() 76 | token_metadata.add_attribute(StringAttribute(name="hat", value="blue cap")) 77 | token_metadata.add_attribute(NumericAttribute(name="integer trait", value=1)) 78 | token_metadata.add_attribute( 79 | DateAttribute(name="created", value=int(datetime.now().timestamp())) 80 | ) 81 | 82 | assert token_metadata.string_attributes == { 83 | "hat": StringAttribute(name="hat", value="blue cap"), 84 | } 85 | assert token_metadata.numeric_attributes == { 86 | "integer trait": NumericAttribute(name="integer trait", value=1), 87 | } 88 | assert token_metadata.date_attributes == { 89 | "created": DateAttribute( 90 | name="created", value=int(datetime.now().timestamp()) 91 | ), 92 | } 93 | 94 | def test_add_attribute_non_empty(self): 95 | created_date = datetime.now() 96 | token_metadata = TokenMetadata.from_attributes( 97 | { 98 | "hat": "blue cap", 99 | "created": created_date, 100 | "integer trait": 1, 101 | "float trait": 203.5, 102 | "PANTS ": "jeans", 103 | } 104 | ) 105 | token_metadata.add_attribute(StringAttribute(name="scarf", value="old")) 106 | token_metadata.add_attribute(StringAttribute(name="scarf", value="wrap-around")) 107 | token_metadata.add_attribute(NumericAttribute(name="integer trait 2", value=10)) 108 | created_date_2 = datetime.now() 109 | token_metadata.add_attribute( 110 | DateAttribute(name="created 2", value=int(created_date_2.timestamp())) 111 | ) 112 | 113 | assert token_metadata.string_attributes == { 114 | "hat": StringAttribute(name="hat", value="blue cap"), 115 | "pants": StringAttribute(name="pants", value="jeans"), 116 | "scarf": StringAttribute(name="scarf", value="wrap-around"), 117 | } 118 | assert token_metadata.numeric_attributes == { 119 | "integer trait": NumericAttribute(name="integer trait", value=1), 120 | "float trait": NumericAttribute(name="float trait", value=203.5), 121 | "integer trait 2": NumericAttribute(name="integer trait 2", value=10), 122 | } 123 | assert token_metadata.date_attributes == { 124 | "created": DateAttribute( 125 | name="created", value=int(created_date.timestamp()) 126 | ), 127 | "created 2": DateAttribute( 128 | name="created 2", value=int(created_date_2.timestamp()) 129 | ), 130 | } 131 | 132 | def test_metadata_to_attributes(self): 133 | attribute_dict = self.token_metadata.to_attributes() 134 | 135 | assert attribute_dict["hat"] == "blue cap" 136 | assert attribute_dict["integer trait"] == 1 137 | 138 | # the microseconds will get lost so compare to a datetime without them 139 | assert attribute_dict["created"] == datetime( 140 | now.year, now.month, now.day, now.hour, now.minute, now.second 141 | ) 142 | -------------------------------------------------------------------------------- /open_rarity/resolver/rarity_providers/trait_sniper.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import time 4 | 5 | import requests 6 | 7 | from .rank_resolver import RankResolver 8 | 9 | logger = logging.getLogger("open_rarity_logger") 10 | # For fetching rank fetches for an entire contract 11 | TRAIT_SNIPER_RANKS_URL = ( 12 | "https://api.traitsniper.com/v1/collections/{contract_address}/ranks" 13 | ) 14 | # For single rank fetches 15 | TRAIT_SNIPER_NFTS_URL = "https://api.traitsniper.com/api/projects/{slug}/nfts" 16 | 17 | USER_AGENT = { 18 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36" # noqa: E501 19 | } 20 | API_KEY = os.environ.get("TRAIT_SNIPER_API_KEY") or "" 21 | 22 | 23 | class TraitSniperResolver(RankResolver): 24 | @staticmethod 25 | def get_all_ranks(contract_address: str) -> dict[str, int]: 26 | """Get all ranks for a contract address 27 | 28 | Returns 29 | ------- 30 | dict[str, int] 31 | A dictionary of token_id to ranks 32 | """ 33 | rank_data_page = TraitSniperResolver.get_ranks(contract_address, page=1) 34 | all_rank_data = rank_data_page 35 | page = 2 36 | 37 | while rank_data_page: 38 | rank_data_page = TraitSniperResolver.get_ranks(contract_address, page=page) 39 | all_rank_data.extend(rank_data_page) 40 | page += 1 41 | # Due to rate limits we need to slow things down a bit... 42 | # Free tier is 5 request per second 43 | time.sleep(12) 44 | 45 | return { 46 | str(rank_data["token_id"]): int(rank_data["rarity_rank"]) 47 | for rank_data in all_rank_data 48 | if rank_data["rarity_rank"] 49 | } 50 | 51 | @staticmethod 52 | def get_ranks(contract_address: str, page: int, limit: int = 200) -> list[dict]: 53 | """ 54 | Parameters 55 | ---------- 56 | contract_address: str 57 | The contract address of collection you're fetching ranks for 58 | limit: int 59 | The number of ranks to fetch. Defaults to 200, and maxes at 200 60 | due to API limitations. 61 | 62 | Returns 63 | ------- 64 | List of rarity rank data from trait sniper API with the following 65 | data structure for each item in the list: 66 | { 67 | "rarity_rank": int, 68 | "rarity_score": float 69 | "token_id": str 70 | } 71 | 72 | Requires 73 | -------- 74 | API KEY 75 | 76 | Raises 77 | ------ 78 | ValueError if contract address is None 79 | """ 80 | if not contract_address: 81 | msg = f"Failed to fetch traitsniper. {contract_address=} is invalid." 82 | logger.exception(msg) 83 | raise ValueError(msg) 84 | 85 | url = TRAIT_SNIPER_RANKS_URL.format(contract_address=contract_address) 86 | headers = { 87 | **USER_AGENT, 88 | **{"X-TS-API-KEY": API_KEY}, 89 | } 90 | query_params = { 91 | "limit": max(limit, 200), 92 | "page": page, 93 | } 94 | response = requests.request("GET", url, headers=headers, params=query_params) 95 | if response.status_code == 200: 96 | return response.json()["ranks"] 97 | else: 98 | if ( 99 | "Collection could not be found on TraitSniper" 100 | in response.json()["message"] 101 | ): 102 | logger.warning( 103 | f"[TraitSniper] Collection not found: {contract_address}" 104 | ) 105 | else: 106 | logger.debug( 107 | "[TraitSniper] Failed to resolve TraitSniper rank for " 108 | f"{contract_address}. Received {response.status_code} " 109 | f"for {url}: {response.reason}. {response.json()}" 110 | ) 111 | return [] 112 | 113 | @staticmethod 114 | def get_rank(collection_slug: str, token_id: int) -> int | None: 115 | """Sends a GET request to Trait Sniper API to fetch ranking 116 | data for a given EVM token. Trait Sniper uses opensea slug as a param. 117 | 118 | Parameters 119 | ---------- 120 | collection_slug : str 121 | collection slug of collection you're attempting to fetch. This must be 122 | the slug on trait sniper's slug system. 123 | token_id : int 124 | the token number. 125 | 126 | Returns 127 | ------- 128 | int | None 129 | Rarity rank for given token ID if request was successful, otherwise None. 130 | 131 | Raises 132 | ------ 133 | ValueError 134 | If slug is invalid. 135 | """ 136 | # TODO [vicky]: In future, we can add retry mechanisms if needed 137 | 138 | querystring: dict[str, str | int] = { 139 | "trait_norm": "true", 140 | "trait_count": "true", 141 | "token_id": token_id, 142 | } 143 | 144 | if not collection_slug: 145 | msg = "Cannot fetch traitsniper rank as slug is None." 146 | logger.exception(msg) 147 | raise ValueError(msg) 148 | 149 | url = TRAIT_SNIPER_NFTS_URL.format(slug=collection_slug) 150 | response = requests.request("GET", url, params=querystring, headers=USER_AGENT) 151 | if response.status_code == 200: 152 | return int(response.json()["nfts"][0]["rarity_rank"]) 153 | else: 154 | logger.debug( 155 | "[TraitSniper] Failed to resolve TraitSniper rank for " 156 | f"{collection_slug} {token_id}. Received {response.status_code} " 157 | f"for {url}: {response.reason}. {response.json()}" 158 | ) 159 | return None 160 | -------------------------------------------------------------------------------- /tests/scoring/test_scoring_handlers.py: -------------------------------------------------------------------------------- 1 | import time 2 | from random import sample 3 | 4 | import numpy as np 5 | import pytest 6 | 7 | from open_rarity.models.collection import TRAIT_COUNT_ATTRIBUTE_NAME, Collection 8 | from open_rarity.scoring.handlers.information_content_scoring_handler import ( 9 | InformationContentScoringHandler, 10 | ) 11 | from open_rarity.scoring.utils import get_token_attributes_scores_and_weights 12 | from tests.helpers import ( 13 | generate_collection_with_token_traits, 14 | generate_mixed_collection, 15 | get_mixed_trait_spread, 16 | onerare_rarity_tokens, 17 | uniform_rarity_tokens, 18 | ) 19 | 20 | 21 | class TestScoringHandlers: 22 | max_scoring_time_for_10k_s = 2 23 | uniform_tokens = uniform_rarity_tokens( 24 | token_total_supply=10_000, attribute_count=5, values_per_attribute=10 25 | ) 26 | 27 | uniform_collection = Collection(tokens=uniform_tokens) 28 | 29 | one_rare_tokens = onerare_rarity_tokens( 30 | token_total_supply=10_000, 31 | attribute_count=3, 32 | values_per_attribute=10, 33 | ) 34 | 35 | # The last token (#9999) has a unique attribute value for all 36 | # 5 different attribute types 37 | onerare_collection = Collection(tokens=one_rare_tokens) 38 | 39 | # Collection with following attribute distribution 40 | # "hat": 41 | # 20% have "cap", 42 | # 30% have "beanie", 43 | # 45% have "hood", 44 | # 5% have "visor" 45 | # "shirt": 46 | # 80% have "white-t", 47 | # 20% have "vest" 48 | # "special": 49 | # 1% have "special" 50 | # others none 51 | mixed_collection = generate_mixed_collection() 52 | 53 | def test_information_content_rarity_uniform(self): 54 | ic_handler = InformationContentScoringHandler() 55 | 56 | uniform_token_to_test = self.uniform_collection.tokens[0] 57 | uniform_ic_rarity = 1.0 58 | assert np.round( 59 | ic_handler.score_token( 60 | collection=self.uniform_collection, token=uniform_token_to_test 61 | ), 62 | 8, 63 | ) == np.round(uniform_ic_rarity, 8) 64 | 65 | def test_information_content_rarity_mixed(self): 66 | ic_scorer = InformationContentScoringHandler() 67 | 68 | # First test collection entropy 69 | collection_entropy = ic_scorer._get_collection_entropy(self.mixed_collection) 70 | collection_probs = [] 71 | mixed_spread = get_mixed_trait_spread() 72 | for trait_dict in mixed_spread.values(): 73 | for tokens_with_trait in trait_dict.values(): 74 | collection_probs.append(tokens_with_trait / 10000) 75 | 76 | assert np.round(collection_entropy, 10) == np.round( 77 | -np.dot(collection_probs, np.log2(collection_probs)), 10 78 | ) 79 | 80 | # Test the actual scores 81 | token_idxs_to_test = sample(range(self.mixed_collection.token_total_supply), 20) 82 | scores = ic_scorer.score_tokens( 83 | collection=self.mixed_collection, 84 | tokens=self.mixed_collection.tokens, 85 | ) 86 | assert len(scores) == 10000 87 | for token_idx in token_idxs_to_test: 88 | token = self.mixed_collection.tokens[token_idx] 89 | score = scores[token_idx] 90 | assert score == ic_scorer.score_token( 91 | collection=self.mixed_collection, 92 | token=token, 93 | ) 94 | attr_scores, _ = get_token_attributes_scores_and_weights( 95 | collection=self.mixed_collection, 96 | token=token, 97 | normalized=True, 98 | ) 99 | ic_token_score = -np.sum(np.log2(np.reciprocal(attr_scores))) 100 | 101 | assert score == ic_token_score / collection_entropy 102 | 103 | def test_information_content_null_value_attribute(self): 104 | ic_scorer = InformationContentScoringHandler() 105 | collection_with_empty = generate_collection_with_token_traits( 106 | [ 107 | {"bottom": "spec", "hat": "spec", "special": "true"}, # trait count = 3 108 | {"bottom": "1", "hat": "1", "special": "true"}, # trait count = 3 109 | {"bottom": "1", "hat": "1"}, # trait count = 2 110 | {"bottom": "2", "hat": "2"}, # trait count = 2 111 | {"bottom": "2", "hat": "2"}, # trait count = 2 112 | {"bottom": "3", "hat": "2"}, # trait count = 2 113 | ] 114 | ) 115 | 116 | collection_entropy = ic_scorer._get_collection_entropy(collection_with_empty) 117 | collection_probs = [] 118 | spread = { 119 | "bottom": {"1": 2, "2": 2, "3": 1, "spec": 1}, 120 | "hat": {"1": 2, "2": 3, "spec": 1}, 121 | "special": {"true": 2, "Null": 4}, 122 | TRAIT_COUNT_ATTRIBUTE_NAME: {"2": 4, "3": 2}, 123 | } 124 | for trait_dict in spread.values(): 125 | for tokens_with_trait in trait_dict.values(): 126 | collection_probs.append(tokens_with_trait / 6) 127 | 128 | expected_collection_entropy = -np.dot( 129 | collection_probs, np.log2(collection_probs) 130 | ) 131 | assert np.round(collection_entropy, 10) == np.round( 132 | expected_collection_entropy, 10 133 | ) 134 | 135 | scores = ic_scorer.score_tokens( 136 | collection=collection_with_empty, tokens=collection_with_empty.tokens 137 | ) 138 | assert scores[0] > scores[1] 139 | assert scores[1] > scores[2] 140 | assert scores[5] > scores[2] 141 | assert scores[2] > scores[3] 142 | assert scores[3] == scores[4] 143 | 144 | for i, token in enumerate(collection_with_empty.tokens): 145 | attr_scores, _ = get_token_attributes_scores_and_weights( 146 | collection=collection_with_empty, 147 | token=token, 148 | normalized=False, 149 | ) 150 | ic_token_score = -np.sum(np.log2(np.reciprocal(attr_scores))) 151 | expected_score = ic_token_score / collection_entropy 152 | 153 | assert np.round(scores[i], 10) == np.round(expected_score, 10) 154 | 155 | def test_information_content_empty_attribute(self): 156 | collection_with_null = generate_collection_with_token_traits( 157 | [ 158 | {"bottom": "1", "hat": "1", "special": "true"}, 159 | {"bottom": "1", "hat": "1"}, 160 | {"bottom": "2", "hat": "2"}, 161 | {"bottom": "2", "hat": "2"}, 162 | {"bottom": "3", "hat": "2"}, 163 | ] 164 | ) 165 | 166 | collection_without_null = generate_collection_with_token_traits( 167 | [ 168 | {"bottom": "1", "hat": "1", "special": "true"}, 169 | {"bottom": "1", "hat": "1", "special": "none"}, 170 | {"bottom": "2", "hat": "2", "special": "none"}, 171 | {"bottom": "2", "hat": "2", "special": "none"}, 172 | {"bottom": "3", "hat": "2", "special": "none"}, 173 | ] 174 | ) 175 | 176 | ic_scorer = InformationContentScoringHandler() 177 | 178 | scores_with_null = ic_scorer.score_tokens( 179 | collection=collection_with_null, tokens=collection_with_null.tokens 180 | ) 181 | scores_without_null = ic_scorer.score_tokens( 182 | collection=collection_without_null, 183 | tokens=collection_without_null.tokens, 184 | ) 185 | 186 | assert scores_with_null == scores_without_null 187 | 188 | @pytest.mark.skip(reason="Not including performance testing as required testing") 189 | def test_information_content_rarity_timing(self): 190 | ic_scorer = InformationContentScoringHandler() 191 | tic = time.time() 192 | ic_scorer.score_tokens( 193 | collection=self.mixed_collection, 194 | tokens=self.mixed_collection.tokens, 195 | ) 196 | toc = time.time() 197 | assert (toc - tic) < self.max_scoring_time_for_10k_s 198 | -------------------------------------------------------------------------------- /open_rarity/models/token_metadata.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from dataclasses import dataclass, field 3 | from typing import Any 4 | 5 | from open_rarity.models.utils.attribute_utils import normalize_attribute_string 6 | 7 | AttributeName = str 8 | AttributeValue = str 9 | 10 | 11 | @dataclass 12 | class StringAttribute: 13 | """Class represents string token attribute name and value 14 | 15 | Attributes 16 | ---------- 17 | name : AttributeName 18 | name of an attribute 19 | value : str 20 | value of a string attribute 21 | """ 22 | 23 | name: AttributeName # duplicate name here for ease of reduce 24 | value: AttributeValue 25 | 26 | def __init__(self, name: AttributeName, value: AttributeValue): 27 | # We treat string attributes name and value the same regardless of 28 | # casing or leading/trailing whitespaces. 29 | self.name = normalize_attribute_string(name) 30 | self.value = normalize_attribute_string(value) 31 | 32 | 33 | @dataclass 34 | class NumericAttribute: 35 | """Class represents numeric token attribute name and value 36 | 37 | Attributes 38 | ---------- 39 | name : AttributeName 40 | name of an attribute 41 | value : float | int 42 | value of a numeric attribute 43 | """ 44 | 45 | name: AttributeName 46 | value: float | int 47 | 48 | def __init__(self, name: AttributeName, value: float | int): 49 | # We treat attributes names the same regardless of 50 | # casing or leading/trailing whitespaces. 51 | self.name = normalize_attribute_string(name) 52 | self.value = value 53 | 54 | 55 | @dataclass 56 | class DateAttribute: 57 | """Class represents date token attribute name and value 58 | 59 | Attributes 60 | ---------- 61 | name : AttributeName 62 | name of an attribute 63 | value : int 64 | value of a date attribute in UNIX timestamp format 65 | """ 66 | 67 | name: AttributeName 68 | value: int 69 | 70 | def __init__(self, name: AttributeName, value: int): 71 | # We treat attributes names the same regardless of 72 | # casing or leading/trailing whitespaces. 73 | self.name = normalize_attribute_string(name) 74 | self.value = value 75 | 76 | 77 | Attribute = StringAttribute | NumericAttribute | DateAttribute 78 | 79 | 80 | @dataclass 81 | class TokenMetadata: 82 | """Class represents EIP-721 or EIP-1115 compatible metadata structure 83 | 84 | Attributes 85 | ---------- 86 | string_attributes : dict 87 | mapping of atrribute name to list of string attribute values 88 | numeric_attributes : dict 89 | mapping of atrribute name to list of numeric attribute values 90 | date_attributes : dict 91 | mapping of attribute name to list of date attribute values 92 | 93 | 94 | All attributes names are normalized and all string attribute values are 95 | normalized in the same way - lowercased and leading/trailing whitespace stripped. 96 | """ 97 | 98 | string_attributes: dict[AttributeName, StringAttribute] = field( 99 | default_factory=dict 100 | ) 101 | numeric_attributes: dict[AttributeName, NumericAttribute] = field( 102 | default_factory=dict 103 | ) 104 | date_attributes: dict[AttributeName, DateAttribute] = field(default_factory=dict) 105 | 106 | def __post_init__(self): 107 | self.string_attributes = self._normalize_attributes_dict(self.string_attributes) 108 | self.numeric_attributes = self._normalize_attributes_dict( 109 | self.numeric_attributes 110 | ) 111 | self.date_attributes = self._normalize_attributes_dict(self.date_attributes) 112 | 113 | @classmethod 114 | def from_attributes(cls, attributes: dict[AttributeName, Any]): 115 | """Constructs TokenMetadata class based on an attributes dictionary 116 | 117 | Parameters 118 | ---------- 119 | attributes : dict[AttributeName, Any] 120 | Dictionary of attribute name to attribute value for the given token. 121 | The type of the value determines whether the attribute is a string, 122 | numeric or date attribute. 123 | 124 | class attribute type 125 | ------------ ------------- 126 | string string attribute 127 | int | float numeric_attribute 128 | datetime date_attribute (stored as timestamp, seconds from epoch) 129 | 130 | Returns 131 | ------- 132 | TokenMetadata 133 | token metadata from input 134 | """ 135 | string_attributes = {} 136 | numeric_attributes = {} 137 | date_attributes = {} 138 | for attr_name, attr_value in attributes.items(): 139 | if isinstance(attr_value, str): 140 | string_attributes[attr_name] = StringAttribute( 141 | name=attr_name, value=attr_value 142 | ) 143 | elif isinstance(attr_value, (float, int)): 144 | numeric_attributes[attr_name] = NumericAttribute( 145 | name=attr_name, value=attr_value 146 | ) 147 | elif isinstance(attr_value, datetime.datetime): 148 | date_attributes[attr_name] = DateAttribute( 149 | name=attr_name, 150 | value=int(attr_value.timestamp()), 151 | ) 152 | else: 153 | raise TypeError( 154 | f"Provided attribute value has invalid type: {type(attr_value)}. " 155 | "Must be either str, float, int or datetime." 156 | ) 157 | 158 | return cls( 159 | string_attributes=string_attributes, 160 | numeric_attributes=numeric_attributes, 161 | date_attributes=date_attributes, 162 | ) 163 | 164 | def to_attributes(self) -> dict[AttributeName, Any]: 165 | """Returns a dictionary of all attributes in this metadata object.""" 166 | attributes: dict[AttributeName, Any] = {} 167 | for str_attr in self.string_attributes.values(): 168 | attributes[str_attr.name] = str_attr.value 169 | for num_attr in self.numeric_attributes.values(): 170 | attributes[num_attr.name] = num_attr.value 171 | for date_attr in self.date_attributes.values(): 172 | attributes[date_attr.name] = datetime.datetime.fromtimestamp( 173 | date_attr.value 174 | ) 175 | return attributes 176 | 177 | def add_attribute(self, attribute: Attribute): 178 | """Adds an attribute to this metadata object, overriding existing 179 | attribute if the normalized attribute name already exists.""" 180 | if isinstance(attribute, StringAttribute): 181 | self.string_attributes[attribute.name] = attribute 182 | elif isinstance(attribute, NumericAttribute): 183 | self.numeric_attributes[attribute.name] = attribute 184 | elif isinstance(attribute, DateAttribute): 185 | self.date_attributes[attribute.name] = attribute 186 | else: 187 | raise TypeError( 188 | f"Provided attribute has invalid type: {type(attribute)}. " 189 | "Must be either StringAttribute, NumericAttribute or DateAttribute." 190 | ) 191 | 192 | def attribute_exists(self, attribute_name: str) -> bool: 193 | """Returns True if this metadata object has an attribute with the given name.""" 194 | attr_name = normalize_attribute_string(attribute_name) 195 | return ( 196 | attr_name in self.string_attributes 197 | or attr_name in self.numeric_attributes 198 | or attr_name in self.date_attributes 199 | ) 200 | 201 | def _normalize_attributes_dict(self, attributes_dict: dict) -> dict: 202 | """Helper function that takes in an attributes dictionary 203 | and normalizes attribute name in the dictionary to ensure all 204 | letters are lower cases and whitespace is stripped. 205 | """ 206 | normalized_attributes_dict = {} 207 | for attribute_name, attr in attributes_dict.items(): 208 | normalized_attr_name = normalize_attribute_string(attribute_name) 209 | normalized_attributes_dict[normalized_attr_name] = attr 210 | if normalized_attr_name != attr.name: 211 | attr.name = normalized_attr_name 212 | return normalized_attributes_dict 213 | -------------------------------------------------------------------------------- /tests/models/test_token.py: -------------------------------------------------------------------------------- 1 | from open_rarity.models.token import Token 2 | from open_rarity.models.token_identifier import ( 3 | EVMContractTokenIdentifier, 4 | SolanaMintAddressTokenIdentifier, 5 | ) 6 | from open_rarity.models.token_metadata import ( 7 | NumericAttribute, 8 | StringAttribute, 9 | TokenMetadata, 10 | ) 11 | from open_rarity.models.token_standard import TokenStandard 12 | from tests.helpers import create_evm_token 13 | 14 | 15 | class TestToken: 16 | metadata = TokenMetadata( 17 | string_attributes={ 18 | "hat": StringAttribute(name="hat", value="blue"), 19 | "shirt": StringAttribute(name="shirt", value="red"), 20 | }, 21 | numeric_attributes={ 22 | "level": NumericAttribute(name="level", value=1), 23 | }, 24 | ) 25 | token = create_evm_token(token_id=1, metadata=metadata) 26 | 27 | def test_token_metadata(self): 28 | assert self.token.metadata == self.metadata 29 | 30 | def test_create_metaplex_non_fungible(self): 31 | token = Token( 32 | token_identifier=SolanaMintAddressTokenIdentifier( 33 | mint_address="AsjdsskDso..." 34 | ), 35 | token_standard=TokenStandard.METAPLEX_NON_FUNGIBLE, 36 | metadata=TokenMetadata.from_attributes({"hat": "cap", "shirt": "blue"}), 37 | ) 38 | token_equal = Token.from_metaplex_non_fungible( 39 | mint_address="AsjdsskDso...", 40 | attributes={"hat": "cap", "shirt": "blue"}, 41 | ) 42 | 43 | assert token == token_equal 44 | 45 | token_not_equal = Token.from_metaplex_non_fungible( 46 | mint_address="DiffMintAddresss...", 47 | attributes={"hat": "cap", "shirt": "blue"}, 48 | ) 49 | 50 | assert token != token_not_equal 51 | 52 | def test_create_erc721(self): 53 | token = Token( 54 | token_identifier=EVMContractTokenIdentifier( 55 | contract_address="0xa3049...", token_id=1 56 | ), 57 | token_standard=TokenStandard.ERC721, 58 | metadata=TokenMetadata.from_attributes({"hat": "cap", "shirt": "blue"}), 59 | ) 60 | token_equal = Token.from_erc721( 61 | contract_address="0xa3049...", 62 | token_id=1, 63 | metadata_dict={"hat": "cap", "shirt": "blue"}, 64 | ) 65 | 66 | assert token == token_equal 67 | 68 | token_not_equal = Token.from_erc721( 69 | contract_address="0xmew...", 70 | token_id=1, 71 | metadata_dict={"hat": "cap", "shirt": "blue"}, 72 | ) 73 | 74 | assert token != token_not_equal 75 | 76 | def test_token_init_metadata_non_matching_attribute_names(self): 77 | token = create_evm_token( 78 | token_id=1, 79 | metadata=TokenMetadata( 80 | string_attributes={ 81 | "hat": StringAttribute(name="big hat", value="blue"), 82 | "shirt": StringAttribute(name=" shirt", value="red"), 83 | } 84 | ), 85 | ) 86 | assert token.metadata.string_attributes == { 87 | "hat": StringAttribute(name="hat", value="blue"), 88 | "shirt": StringAttribute(name="shirt", value="red"), 89 | } 90 | 91 | def test_token_attribute_normalization(self): 92 | expected_equal_metadata_tokens = [ 93 | create_evm_token( 94 | token_id=1, 95 | metadata=TokenMetadata( 96 | string_attributes={ 97 | "hat ": StringAttribute(name="hat", value="blue"), 98 | "Shirt ": StringAttribute(name="shirt", value="red"), 99 | }, 100 | numeric_attributes={ 101 | "level": NumericAttribute(name="level", value=1), 102 | }, 103 | ), 104 | ), 105 | create_evm_token( 106 | token_id=1, 107 | metadata=TokenMetadata( 108 | string_attributes={ 109 | "hat": StringAttribute(name="hat", value="blue"), 110 | "Shirt ": StringAttribute(name=" shirt", value="red"), 111 | }, 112 | numeric_attributes={ 113 | "Level": NumericAttribute(name="level", value=1), 114 | }, 115 | ), 116 | ), 117 | create_evm_token( 118 | token_id=1, 119 | metadata=TokenMetadata( 120 | string_attributes={ 121 | "Hat": StringAttribute(name=" hat ", value="blue"), 122 | "shirt": StringAttribute(name="shirt", value="red"), 123 | }, 124 | numeric_attributes={ 125 | "Level": NumericAttribute(name=" level ", value=1), 126 | }, 127 | ), 128 | ), 129 | create_evm_token( 130 | token_id=1, 131 | metadata=TokenMetadata( 132 | string_attributes={ 133 | " hat ": StringAttribute(name=" hat ", value="blue"), 134 | " shirt": StringAttribute(name="shirt", value="red"), 135 | }, 136 | numeric_attributes={ 137 | "level": NumericAttribute(name="level ", value=1), 138 | }, 139 | ), 140 | ), 141 | ] 142 | 143 | assert all( 144 | t.metadata == expected_equal_metadata_tokens[0].metadata 145 | for t in expected_equal_metadata_tokens 146 | ) 147 | 148 | expected_not_equal = [ 149 | create_evm_token( 150 | token_id=1, 151 | metadata=TokenMetadata( 152 | string_attributes={ 153 | " big hat ": StringAttribute(name=" hat ", value="blue"), 154 | " shirt": StringAttribute(name="shirt", value="red"), 155 | }, 156 | numeric_attributes={ 157 | "level": NumericAttribute(name="level", value=1), 158 | }, 159 | ), 160 | ), 161 | create_evm_token( 162 | token_id=1, 163 | metadata=TokenMetadata( 164 | string_attributes={ 165 | "hat": StringAttribute(name="hat", value="blue"), 166 | "shirt": StringAttribute(name="shirt", value="red"), 167 | }, 168 | numeric_attributes={ 169 | "big level": NumericAttribute(name="level", value=1), 170 | }, 171 | ), 172 | ), 173 | ] 174 | 175 | assert all( 176 | t.metadata != expected_equal_metadata_tokens[0].metadata 177 | for t in expected_not_equal 178 | ) 179 | 180 | def test_has_attribute(self): 181 | assert self.token.has_attribute("hat") 182 | assert self.token.has_attribute("shirt") 183 | assert self.token.has_attribute("level") 184 | assert not self.token.has_attribute("not an attribute") 185 | 186 | def test_trait_count(self): 187 | assert self.token.trait_count() == 3 188 | non_null_traits = {"hat": "cap", "shirt": "blue", "level": 1, "size": "large"} 189 | 190 | token_with_none = create_evm_token( 191 | token_id=1, 192 | metadata=TokenMetadata.from_attributes( 193 | {**non_null_traits, "something": "none"} 194 | ), 195 | ) 196 | assert token_with_none.trait_count() == 4 197 | 198 | token_with_none = create_evm_token( 199 | token_id=1, 200 | metadata=TokenMetadata.from_attributes( 201 | {**non_null_traits, "something": ""} 202 | ), 203 | ) 204 | assert token_with_none.trait_count() == 4 205 | 206 | token_with_0 = create_evm_token( 207 | token_id=1, 208 | metadata=TokenMetadata.from_attributes({**non_null_traits, "something": 0}), 209 | ) 210 | assert token_with_0.trait_count() == 5 211 | 212 | token_with_valid_null = create_evm_token( 213 | token_id=1, 214 | metadata=TokenMetadata.from_attributes( 215 | {**non_null_traits, "something": "null"} 216 | ), 217 | ) 218 | assert token_with_valid_null.trait_count() == 5 219 | -------------------------------------------------------------------------------- /tests/resolver/test_testset_resolver.py: -------------------------------------------------------------------------------- 1 | import csv 2 | 3 | import pytest 4 | 5 | from open_rarity.resolver.models.token_with_rarity_data import RankProvider 6 | from open_rarity.resolver.testset_resolver import resolve_collection_data 7 | 8 | 9 | class TestTestsetResolver: 10 | # Note: Official ranking is rarity tools (linked by site) 11 | bayc_token_ids_to_ranks = { 12 | # Rare token, official rank=1 13 | 7495: { 14 | # https://app.traitsniper.com/boredapeyachtclub?view=7495 15 | RankProvider.TRAITS_SNIPER: "1", 16 | # https://raritysniffer.com/viewcollection/boredapeyachtclub?nft=7495 17 | RankProvider.RARITY_SNIFFER: "3", 18 | # https://raritysniper.com/bored-ape-yacht-club/7495 19 | RankProvider.RARITY_SNIPER: "1", 20 | RankProvider.OR_INFORMATION_CONTENT: "1", 21 | }, 22 | # Middle token, official rank=3503 23 | 509: { 24 | # https://app.traitsniper.com/boredapeyachtclub?view=509 25 | RankProvider.TRAITS_SNIPER: "2730", 26 | # https://raritysniffer.com/viewcollection/boredapeyachtclub?nft=509 27 | RankProvider.RARITY_SNIFFER: "3257", 28 | # https://raritysniper.com/bored-ape-yacht-club/509 29 | RankProvider.RARITY_SNIPER: "3402", 30 | RankProvider.OR_INFORMATION_CONTENT: "4091", 31 | }, 32 | # Common token, official rank=7623 33 | 8002: { 34 | # https://app.traitsniper.com/boredapeyachtclub?view=8002 35 | RankProvider.TRAITS_SNIPER: "6271", 36 | # https://raritysniffer.com/viewcollection/boredapeyachtclub?nft=8002 37 | RankProvider.RARITY_SNIFFER: "7709", 38 | # https://raritysniper.com/bored-ape-yacht-club/8002 39 | RankProvider.RARITY_SNIPER: "6924", 40 | RankProvider.OR_INFORMATION_CONTENT: "7347", 41 | }, 42 | } 43 | ic_bayc_token_ids_to_ranks = { 44 | "2100": 10, 45 | "5757": 11, 46 | "7754": 12, 47 | "5020": 3101, 48 | "1730": 3102, 49 | "6784": 7344, 50 | "5949": 7345, 51 | "2103": 5414, 52 | "980": 5415, 53 | "5525": 9999, 54 | "5777": 10000, 55 | } 56 | 57 | EXPECTED_COLUMNS = [ 58 | "slug", 59 | "token_id", 60 | "traits_sniper", 61 | "rarity_sniffer", 62 | "rarity_sniper", 63 | "arithmetic", 64 | "geometric", 65 | "harmonic", 66 | "sum", 67 | "information_content", 68 | ] 69 | 70 | @pytest.mark.skipif( 71 | "not config.getoption('--run-resolvers')", 72 | reason="This tests runs too long to have as part of CI/CD but should be " 73 | "run whenver someone changes resolver", 74 | ) 75 | def test_resolve_collection_data_two_providers(self): 76 | # Have the resolver pull in BAYC rarity rankings from various sources 77 | # Just do a check to ensure the ranks from different providers are 78 | # as expected 79 | resolve_collection_data( 80 | resolve_remote_rarity=True, 81 | package_path="tests", 82 | # We exclude trait sniper due to API key requirements 83 | external_rank_providers=[ 84 | RankProvider.RARITY_SNIFFER, 85 | RankProvider.RARITY_SNIPER, 86 | ], 87 | filename="resolver/sample_files/bayc.json", 88 | ) 89 | # Read the file and verify columns values are as expected for the given tokens 90 | output_filename = "testset_boredapeyachtclub.csv" 91 | 92 | rows = 0 93 | with open(output_filename) as csvfile: 94 | resolver_output_reader = csv.reader(csvfile) 95 | for idx, row in enumerate(resolver_output_reader): 96 | rows += 1 97 | if idx == 0: 98 | assert row[0:10] == self.EXPECTED_COLUMNS 99 | else: 100 | token_id = int(row[1]) 101 | if token_id in self.bayc_token_ids_to_ranks: 102 | assert row[0] == "boredapeyachtclub" 103 | expected_ranks = self.bayc_token_ids_to_ranks[token_id] 104 | assert row[3] == expected_ranks[RankProvider.RARITY_SNIFFER] 105 | assert row[4] == expected_ranks[RankProvider.RARITY_SNIPER] 106 | assert row[5] == expected_ranks[RankProvider.OR_ARITHMETIC] 107 | assert ( 108 | row[9] 109 | == expected_ranks[RankProvider.OR_INFORMATION_CONTENT] 110 | ) 111 | 112 | assert rows == 10_001 113 | 114 | @pytest.mark.skipif( 115 | "not config.getoption('--run-resolvers')", 116 | reason="This tests runs too long to have as part of CI/CD but should be " 117 | "run whenever someone changes resolver and requires TRAIT_SNIPER_API_KEY", 118 | ) 119 | # Warning: Trait Sniper ranks are not stable, so this test may fail. 120 | # If it does, just update the expected values to be what's on the site. 121 | def test_resolve_collection_data_traits_sniper(self): 122 | # Have the resolver pull in BAYC rarity rankings from various sources 123 | # Just do a check to ensure the ranks from different providers are 124 | # as expected 125 | resolve_collection_data( 126 | resolve_remote_rarity=True, 127 | package_path="tests", 128 | external_rank_providers=[RankProvider.TRAITS_SNIPER], 129 | filename="resolver/sample_files/bayc.json", 130 | ) 131 | # Read the file and verify columns values are as expected for the given tokens 132 | output_filename = "testset_boredapeyachtclub.csv" 133 | 134 | rows = 0 135 | with open(output_filename) as csvfile: 136 | resolver_output_reader = csv.reader(csvfile) 137 | for idx, row in enumerate(resolver_output_reader): 138 | rows += 1 139 | if idx == 0: 140 | assert row[0:10] == self.EXPECTED_COLUMNS 141 | else: 142 | token_id = int(row[1]) 143 | if token_id in self.bayc_token_ids_to_ranks: 144 | assert row[0] == "boredapeyachtclub" 145 | expected_ranks = self.bayc_token_ids_to_ranks[token_id] 146 | assert row[2] == expected_ranks[RankProvider.TRAITS_SNIPER] 147 | assert row[5] == expected_ranks[RankProvider.OR_ARITHMETIC] 148 | assert ( 149 | row[9] 150 | == expected_ranks[RankProvider.OR_INFORMATION_CONTENT] 151 | ) 152 | 153 | assert rows == 10_001 154 | 155 | @pytest.mark.skipif( 156 | "not config.getoption('--run-resolvers')", 157 | reason="This tests depends on external API", 158 | ) 159 | def test_resolve_collection_data_rarity_sniffer(self): 160 | # Have the resolver pull in BAYC rarity rankings from various sources 161 | # Just do a check to ensure the ranks from different providers are 162 | # as expected 163 | slug_to_rows = resolve_collection_data( 164 | resolve_remote_rarity=True, 165 | package_path="tests", 166 | external_rank_providers=[RankProvider.RARITY_SNIFFER], 167 | filename="resolver/sample_files/bayc.json", 168 | output_file_to_disk=False, 169 | ) 170 | rows = slug_to_rows["boredapeyachtclub"] 171 | 172 | for row in rows: 173 | token_id = int(row[1]) 174 | if token_id in self.bayc_token_ids_to_ranks: 175 | assert row[0] == "boredapeyachtclub" 176 | expected_ranks = self.bayc_token_ids_to_ranks[token_id] 177 | assert str(row[3]) == expected_ranks[RankProvider.RARITY_SNIFFER] 178 | 179 | assert len(rows) == 10_000 180 | 181 | @pytest.mark.skipif( 182 | "not config.getoption('--run-resolvers')", 183 | reason="Needs OpenSea API key to be set up", 184 | ) 185 | def test_resolve_collection_data_no_external(self): 186 | # Have the resolver pull in BAYC rarity rankings from various sources 187 | # Just do a check to ensure the ranks from different providers are 188 | # as expected 189 | resolve_collection_data( 190 | resolve_remote_rarity=True, 191 | package_path="tests", 192 | external_rank_providers=[], 193 | filename="resolver/sample_files/bayc.json", 194 | ) 195 | # Read the file and verify columns values are as expected for the given tokens 196 | output_filename = "testset_boredapeyachtclub.csv" 197 | 198 | rows = 0 199 | with open(output_filename) as csvfile: 200 | resolver_output_reader = csv.reader(csvfile) 201 | for idx, row in enumerate(resolver_output_reader): 202 | rows += 1 203 | if idx == 0: 204 | assert row[0:10] == self.EXPECTED_COLUMNS 205 | else: 206 | token_id = int(row[1]) 207 | if token_id in self.bayc_token_ids_to_ranks: 208 | assert row[0] == "boredapeyachtclub" 209 | expected_ranks = self.bayc_token_ids_to_ranks[token_id] 210 | assert ( 211 | row[9] 212 | == expected_ranks[RankProvider.OR_INFORMATION_CONTENT] 213 | ) 214 | if token_id in self.ic_bayc_token_ids_to_ranks: 215 | assert row[9] == self.ic_bayc_token_ids_to_ranks[token_id] 216 | 217 | assert rows == 10_001 218 | -------------------------------------------------------------------------------- /open_rarity/scoring/handlers/information_content_scoring_handler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import numpy as np 4 | 5 | from open_rarity.models.collection import Collection, CollectionAttribute 6 | from open_rarity.models.token import Token 7 | from open_rarity.models.token_metadata import AttributeName 8 | from open_rarity.scoring.utils import get_token_attributes_scores_and_weights 9 | 10 | logger = logging.getLogger("open_rarity_logger") 11 | 12 | 13 | class InformationContentScoringHandler: 14 | """Rarity describes the information-theoretic "rarity" of a Collection. 15 | The concept of "rarity" can be considered as a measure of "surprise" at the 16 | occurrence of a particular token's properties, within the context of the 17 | Collection from which it is derived. Self-information is a measure of such 18 | surprise, and information entropy a measure of the expected value of 19 | self-information across a distribution (i.e. across a Collection). 20 | 21 | It is trivial to "stuff" a Collection with extra information by merely adding 22 | additional properties to all tokens. This is reflected in the Entropy field, 23 | measured in bits—all else held equal, a Collection with more token properties 24 | will have higher Entropy. However, this information bloat is carried by the 25 | tokens themselves, so their individual information-content grows in line with 26 | Collection-wide Entropy. The Scores are therefore scaled down by the Entropy 27 | to provide unitless "relative surprise", which can be safely compared between 28 | Collections. 29 | 30 | This class computes rarity of each token in the Collection based on information 31 | entropy. Every TraitType is considered as a categorical probability 32 | distribution with each TraitValue having an associated probability and hence 33 | information content. The rarity of a particular token is the sum of 34 | information content carried by each of its Attributes, divided by the entropy 35 | of the Collection as a whole (see the Rarity struct for rationale). 36 | 37 | Notably, the lack of a TraitType is considered as a null-Value Attribute as 38 | the absence across the majority of a Collection implies rarity in those 39 | tokens that do carry the TraitType. 40 | """ 41 | 42 | # TODO [@danmeshkov]: To support numeric types in a follow-up version. 43 | 44 | def score_token(self, collection: Collection, token: Token) -> float: 45 | """See ScoringHandler interface. 46 | 47 | Limitations 48 | ----------- 49 | Does not take into account non-String attributes during scoring. 50 | 51 | """ 52 | return self._score_token(collection, token) 53 | 54 | def score_tokens( 55 | self, 56 | collection: Collection, 57 | tokens: list[Token], 58 | ) -> list[float]: 59 | """See ScoringHandler interface. 60 | 61 | Limitations 62 | ----------- 63 | Does not take into account non-String attributes during scoring. 64 | 65 | """ 66 | # Precompute for performance 67 | collection_null_attributes = collection.extract_null_attributes() 68 | collection_attributes = collection.extract_collection_attributes() 69 | collection_entropy = self._get_collection_entropy( 70 | collection=collection, 71 | collection_attributes=collection_attributes, 72 | collection_null_attributes=collection_null_attributes, 73 | ) 74 | return [ 75 | self._score_token( 76 | collection=collection, 77 | token=t, 78 | collection_null_attributes=collection_null_attributes, 79 | # covering the corner case when collection has one item. 80 | collection_entropy_normalization=collection_entropy 81 | if collection_entropy 82 | else 1, 83 | ) 84 | for t in tokens 85 | ] 86 | 87 | # Private methods 88 | def _score_token( 89 | self, 90 | collection: Collection, 91 | token: Token, 92 | collection_null_attributes: dict[AttributeName, CollectionAttribute] = None, 93 | collection_entropy_normalization: float = None, 94 | ) -> float: 95 | """Calculates the score of the token using information entropy with a 96 | collection entropy normalization factor. 97 | 98 | Parameters 99 | ---------- 100 | collection : Collection 101 | The collection with the attributes frequency counts to base the 102 | token trait probabilities on to calculate score. 103 | token : Token 104 | The token to score 105 | collection_null_attributes : dict[AttributeName, CollectionAttribute], optional 106 | Optional memoization of collection.extract_null_attributes(), 107 | by default None. 108 | collection_entropy_normalization : float, optional 109 | Optional memoization of the collection entropy normalization factor, 110 | by default None. 111 | 112 | Returns 113 | ------- 114 | float 115 | The token score 116 | """ 117 | logger.debug("Computing score for token %s", token) 118 | 119 | ic_token_score = self._get_ic_score( 120 | collection, token, collection_null_attributes=collection_null_attributes 121 | ) 122 | logger.debug("IC token score %s", ic_token_score) 123 | 124 | # Now, calculate the collection entropy to use as a normalization for 125 | # the token score if its not provided 126 | if collection_entropy_normalization is None: 127 | collection_entropy = self._get_collection_entropy( 128 | collection=collection, 129 | collection_attributes=collection.extract_collection_attributes(), 130 | collection_null_attributes=collection_null_attributes, 131 | ) 132 | else: 133 | collection_entropy = collection_entropy_normalization 134 | normalized_token_score = ic_token_score / collection_entropy 135 | logger.debug( 136 | "Finished scoring %s %s: collection entropy: %s token scores: %s", 137 | collection, 138 | token, 139 | collection_entropy, 140 | normalized_token_score, 141 | ) 142 | 143 | return normalized_token_score 144 | 145 | def _get_ic_score( 146 | self, 147 | collection: Collection, 148 | token: Token, 149 | collection_null_attributes: dict[AttributeName, CollectionAttribute] = None, 150 | ) -> float: 151 | # First calculate the individual attribute scores for all attributes 152 | # of the provided token. Scores are the inverted probabilities of the 153 | # attribute in the collection. 154 | attr_scores, _ = get_token_attributes_scores_and_weights( 155 | collection=collection, 156 | token=token, 157 | normalized=False, 158 | collection_null_attributes=collection_null_attributes, 159 | ) 160 | 161 | # Get a single score (via information content) for the token by taking 162 | # the sum of the logarithms of the attributes' scores. 163 | return -np.sum(np.log2(np.reciprocal(attr_scores))) 164 | 165 | def _get_collection_entropy( 166 | self, 167 | collection: Collection, 168 | collection_attributes: dict[AttributeName, list[CollectionAttribute]] = None, 169 | collection_null_attributes: dict[AttributeName, CollectionAttribute] = None, 170 | ) -> float: 171 | """Calculates the entropy of the collection, defined to be the 172 | sum of the probability of every possible attribute name/value pair that 173 | occurs in the collection times that square root of such probability. 174 | 175 | Parameters 176 | ---------- 177 | collection : Collection 178 | The collection to calculate probability on 179 | collection_attributes : dict[AttributeName, list[CollectionAttribute]], optional 180 | Optional memoization of collection.extract_collection_attributes(), 181 | by default None. 182 | collection_null_attributes : dict[AttributeName, CollectionAttribute], optional 183 | Optional memoization of collection.extract_null_attributes(), 184 | by default None. 185 | 186 | Returns 187 | ------- 188 | float 189 | the collection entropy 190 | 191 | Limitations 192 | ----------- 193 | Does not take into account non-String attributes during scoring. 194 | """ 195 | attributes: dict[str, list[CollectionAttribute]] = ( 196 | collection_attributes or collection.extract_collection_attributes() 197 | ) 198 | null_attributes: dict[str, CollectionAttribute] = ( 199 | collection_null_attributes or collection.extract_null_attributes() 200 | ) 201 | 202 | # Create a list of all probabilities for every attribute name/value pair, 203 | # in the order of collection.attributes_frequency_counts.items() 204 | collection_probabilities: list[float] = [] 205 | for attr_name, attr_values in attributes.items(): 206 | if attr_name in null_attributes: 207 | null_attr = null_attributes[attr_name] 208 | attr_values.append(null_attr) 209 | 210 | # Create an array of the probability of all possible attr_name/value combos 211 | # existing in the collection 212 | collection_probabilities.extend( 213 | [ 214 | attr_value.total_tokens / collection.token_total_supply 215 | for attr_value in attr_values 216 | ] 217 | ) 218 | 219 | logger.debug("Calculated collection probabilties: %s", collection_probabilities) 220 | collection_entropy = -np.dot( 221 | collection_probabilities, np.log2(collection_probabilities) 222 | ) 223 | 224 | return collection_entropy 225 | -------------------------------------------------------------------------------- /tests/scoring/test_utils.py: -------------------------------------------------------------------------------- 1 | from random import sample 2 | 3 | from open_rarity.models.collection import TRAIT_COUNT_ATTRIBUTE_NAME 4 | from open_rarity.scoring.utils import get_token_attributes_scores_and_weights 5 | from tests.helpers import ( 6 | generate_collection_with_token_traits, 7 | generate_mixed_collection, 8 | generate_onerare_rarity_collection, 9 | generate_uniform_rarity_collection, 10 | get_mixed_trait_spread, 11 | ) 12 | 13 | 14 | class TestScoringUtils: 15 | mixed_collection = generate_mixed_collection() 16 | 17 | def test_get_token_attributes_scores_and_weights_timing_single(self): 18 | import time 19 | 20 | collection = self.mixed_collection 21 | start_time = time.time() 22 | get_token_attributes_scores_and_weights( 23 | collection=collection, 24 | token=collection.tokens[0], 25 | normalized=True, 26 | ) 27 | end_time = time.time() 28 | time_taken = end_time - start_time 29 | print(f"This is single time in seconds: {time_taken}") 30 | assert time_taken < 0.003 31 | 32 | def test_get_token_attributes_scores_and_weights_timing_avg(self): 33 | import time 34 | 35 | collection = self.mixed_collection 36 | tokens_to_test = sample(collection.tokens, 20) 37 | start_time = time.time() 38 | for token in tokens_to_test: 39 | get_token_attributes_scores_and_weights( 40 | collection=collection, 41 | token=token, 42 | normalized=True, 43 | ) 44 | end_time = time.time() 45 | avg_time = (end_time - start_time) / 20 46 | print(f"This is avg time in seconds: {avg_time}") 47 | assert avg_time < 0.003 48 | 49 | def test_get_token_attributes_scores_and_weights_uniform(self): 50 | uniform_collection = generate_uniform_rarity_collection( 51 | attribute_count=5, 52 | values_per_attribute=10, 53 | token_total_supply=10000, 54 | ) 55 | uniform_tokens_to_test = [ 56 | uniform_collection.tokens[0], 57 | uniform_collection.tokens[1405], 58 | uniform_collection.tokens[9999], 59 | ] 60 | # Note: Since trait count is automatically added, attributes are actually 6 61 | for token_to_test in uniform_tokens_to_test: 62 | scores, weights = get_token_attributes_scores_and_weights( 63 | collection=uniform_collection, 64 | token=token_to_test, 65 | normalized=True, 66 | ) 67 | assert scores == [10] * 5 + [1.0] 68 | assert weights == [0.10] * 5 + [1.0] 69 | 70 | scores, weights = get_token_attributes_scores_and_weights( 71 | collection=uniform_collection, 72 | token=token_to_test, 73 | normalized=False, 74 | ) 75 | assert scores == [10] * 5 + [1.0] 76 | assert weights == [1] * 6 77 | 78 | def test_get_token_attributes_scores_and_weights_one_rare(self): 79 | # The last token (#9999) has a unique attribute value for all 80 | # 5 different attribute types 81 | onerare_collection = generate_onerare_rarity_collection( 82 | attribute_count=3, 83 | values_per_attribute=10, 84 | token_total_supply=10000, 85 | ) 86 | # Verify common tokens 87 | common_tokens_to_test = [ 88 | onerare_collection.tokens[0], 89 | onerare_collection.tokens[5045], 90 | onerare_collection.tokens[9998], 91 | ] 92 | # Scores for common tokens are around ~9.0009 93 | for token_to_test in common_tokens_to_test: 94 | scores, weights = get_token_attributes_scores_and_weights( 95 | collection=onerare_collection, 96 | token=token_to_test, 97 | normalized=True, 98 | ) 99 | assert scores == [10000 / 1111] * 3 + [1.0] 100 | assert weights == [0.10] * 3 + [1.0] 101 | 102 | scores, weights = get_token_attributes_scores_and_weights( 103 | collection=onerare_collection, 104 | token=token_to_test, 105 | normalized=False, 106 | ) 107 | assert scores == [10000 / 1111] * 3 + [1.0] 108 | assert weights == [1] * 4 109 | 110 | # Verify the one rare token score 111 | rare_token = onerare_collection.tokens[9999] 112 | scores, weights = get_token_attributes_scores_and_weights( 113 | collection=onerare_collection, 114 | token=rare_token, 115 | normalized=True, 116 | ) 117 | assert scores == [10000 / 1] * 3 + [1.0] 118 | assert weights == [0.10] * 3 + [1.0] 119 | 120 | def test_get_token_attributes_scores_and_weights_scores_vary(self): 121 | collection = generate_collection_with_token_traits( 122 | [ 123 | {"bottom": "1", "hat": "1"}, 124 | {"bottom": "1", "hat": "1"}, 125 | {"bottom": "1", "hat": "2"}, 126 | ] 127 | ) 128 | expected_scores = [ 129 | [3 / 3, 3 / 2, 1.0], 130 | [3 / 3, 3 / 2, 1.0], 131 | [3 / 3, 3 / 1, 1.0], 132 | ] 133 | 134 | for i in range(collection.token_total_supply): 135 | scores, weights = get_token_attributes_scores_and_weights( 136 | collection=collection, 137 | token=collection.tokens[i], 138 | normalized=True, 139 | ) 140 | assert scores == expected_scores[i] 141 | assert weights == [1, 0.5, 1] 142 | 143 | def test_get_token_attributes_scores_and_weights_score_mix(self): 144 | mixed_collection = self.mixed_collection 145 | tokens_to_test = sample(mixed_collection.tokens, 20) 146 | trait_spread = { 147 | **get_mixed_trait_spread(), 148 | TRAIT_COUNT_ATTRIBUTE_NAME: {"3": 10_000}, 149 | } 150 | for token in tokens_to_test: 151 | scores, weights = get_token_attributes_scores_and_weights( 152 | collection=mixed_collection, 153 | token=token, 154 | normalized=True, 155 | ) 156 | expected_scores = [] 157 | expected_weights = [] 158 | for attribute_name, str_attribute in sorted( 159 | token.metadata.string_attributes.items() 160 | ): 161 | num_tokens_with_trait = trait_spread[attribute_name][ 162 | str_attribute.value 163 | ] 164 | expected_scores.append(10000 / num_tokens_with_trait) 165 | expected_weights.append(1 / len(trait_spread[attribute_name])) 166 | 167 | assert scores == expected_scores 168 | assert weights == expected_weights 169 | 170 | def test_get_token_attributes_scores_and_weights_empty_attributes(self): 171 | collection_with_null = generate_collection_with_token_traits( 172 | [ 173 | {"bottom": "1", "hat": "1", "special": "true"}, 174 | {"bottom": "1", "hat": "1"}, 175 | {"bottom": "2", "hat": "2"}, 176 | {"bottom": "2", "hat": "2"}, 177 | {"bottom": "3", "hat": "2"}, 178 | ] 179 | ) 180 | expected_weights_with_null = [1 / 3, 1 / 2, 1 / 2, 1] 181 | 182 | collection_without_null = generate_collection_with_token_traits( 183 | [ 184 | {"bottom": "1", "hat": "1", "special": "true"}, 185 | {"bottom": "1", "hat": "1", "special": "none"}, 186 | {"bottom": "2", "hat": "2", "special": "none"}, 187 | {"bottom": "2", "hat": "2", "special": "none"}, 188 | {"bottom": "3", "hat": "2", "special": "none"}, 189 | ] 190 | ) 191 | expected_weights_without_null = [1 / 3, 1 / 2, 1 / 2, 1 / 2] 192 | expected_scores = [ 193 | [5 / 2, 5 / 2, 5 / 1, 5 / 1], 194 | [5 / 2, 5 / 2, 5 / 4, 5 / 4], 195 | [5 / 2, 5 / 3, 5 / 4, 5 / 4], 196 | [5 / 2, 5 / 3, 5 / 4, 5 / 4], 197 | [5 / 1, 5 / 3, 5 / 4, 5 / 4], 198 | ] 199 | 200 | for collection, expected_weights in [ 201 | [collection_with_null, expected_weights_with_null], 202 | [collection_without_null, expected_weights_without_null], 203 | ]: 204 | for i in range(collection.token_total_supply): 205 | scores, weights = get_token_attributes_scores_and_weights( 206 | collection=collection, 207 | token=collection.tokens[i], 208 | normalized=True, 209 | ) 210 | assert scores == expected_scores[i] 211 | assert weights == expected_weights 212 | 213 | def test_get_token_attributes_scores_and_weights_null_attributes(self): 214 | collection_with_null_value = generate_collection_with_token_traits( 215 | [ 216 | {"bottom": "1", "hat": "1", "special": "true"}, 217 | {"bottom": "1", "hat": "1", "special": "null"}, 218 | {"bottom": "2", "hat": "2", "special": "null"}, 219 | {"bottom": "2", "hat": "2", "special": "null"}, 220 | {"bottom": "3", "hat": "2", "special": "null"}, 221 | ] 222 | ) 223 | expected_weights = [1 / 3, 1 / 2, 1, 1 / 2] 224 | expected_scores = [ 225 | [5 / 2, 5 / 2, 1, 5 / 1], 226 | [5 / 2, 5 / 2, 1, 5 / 4], 227 | [5 / 2, 5 / 3, 1, 5 / 4], 228 | [5 / 2, 5 / 3, 1, 5 / 4], 229 | [5 / 1, 5 / 3, 1, 5 / 4], 230 | ] 231 | 232 | for i in range(collection_with_null_value.token_total_supply): 233 | scores, weights = get_token_attributes_scores_and_weights( 234 | collection=collection_with_null_value, 235 | token=collection_with_null_value.tokens[i], 236 | normalized=True, 237 | ) 238 | assert scores == expected_scores[i] 239 | assert weights == expected_weights 240 | -------------------------------------------------------------------------------- /tests/test_rarity_ranker.py: -------------------------------------------------------------------------------- 1 | from open_rarity.models.collection import Collection 2 | from open_rarity.models.token import Token 3 | from open_rarity.models.token_identifier import ( 4 | EVMContractTokenIdentifier, 5 | SolanaMintAddressTokenIdentifier, 6 | ) 7 | from open_rarity.models.token_metadata import TokenMetadata 8 | from open_rarity.models.token_ranking_features import TokenRankingFeatures 9 | from open_rarity.models.token_rarity import TokenRarity 10 | from open_rarity.rarity_ranker import RarityRanker 11 | from open_rarity.scoring.scorer import Scorer 12 | from tests.helpers import generate_collection_with_token_traits 13 | 14 | 15 | def verify_token_rarities(token_rarities: list[TokenRarity], expected_data: list[dict]): 16 | """ 17 | Parameters 18 | ---------- 19 | expected_data: list[dict] 20 | must be a list of dicts with the following keys: 21 | - id (token id) 22 | - unique_traits 23 | - rank 24 | - score (optional) 25 | """ 26 | for token_rarity, expected in zip(token_rarities, expected_data): 27 | assert token_rarity.rank == expected["rank"] 28 | assert token_rarity.token.token_identifier.token_id == expected["id"] 29 | assert ( 30 | token_rarity.token_features.unique_attribute_count 31 | == expected["unique_traits"] 32 | ) 33 | if "score" in expected: 34 | assert token_rarity.score == expected["score"] 35 | 36 | 37 | class TestRarityRanker: 38 | def test_rarity_ranker_empty_collection(self) -> None: 39 | assert RarityRanker.rank_collection(collection=None) == [] 40 | assert ( 41 | RarityRanker.rank_collection( 42 | collection=Collection(attributes_frequency_counts={}, tokens=[]) 43 | ) 44 | == [] 45 | ) 46 | 47 | def test_rarity_ranker_one_item(self) -> None: 48 | test_collection: Collection = generate_collection_with_token_traits( 49 | [{"trait1": "value1"}] # Token 0 50 | ) 51 | 52 | tokens: list[TokenRarity] = RarityRanker.rank_collection( 53 | collection=test_collection 54 | ) 55 | 56 | assert tokens[0].score == 0 57 | assert tokens[0].rank == 1 58 | 59 | def test_rank_solana_collection(self) -> None: 60 | test_collection = generate_collection_with_token_traits( 61 | [{"trait1": "value1"}], 62 | token_identifier_type=SolanaMintAddressTokenIdentifier.identifier_type, 63 | ) 64 | tokens: list[TokenRarity] = RarityRanker.rank_collection( 65 | collection=test_collection 66 | ) 67 | 68 | assert tokens[0].score == 0 69 | assert tokens[0].rank == 1 70 | 71 | def test_rarity_ranker_equal_score_and_unique_trait(self) -> None: 72 | test_collection = generate_collection_with_token_traits( 73 | [ 74 | # Token 0 75 | { 76 | "trait1": "value1", # 100% 77 | "trait2": "value1", # unique trait 78 | "trait3": "value2", # 75% 79 | }, 80 | # Token 1 81 | { 82 | "trait1": "value1", # 75% 83 | "trait2": "value2", # 75% 84 | "trait3": "value2", # 50% 85 | }, 86 | # Token 2 87 | { 88 | "trait1": "value1", # 100% 89 | "trait2": "value2", # 75% 90 | "trait3": "value3", # unique trait 91 | }, 92 | # Token 3 93 | { 94 | "trait1": "value1", # 100% 95 | "trait2": "value2", # 75% 96 | "trait3": "value2", # 75% 97 | "trait4": "value1", # unique trait 98 | }, 99 | ] 100 | ) 101 | 102 | token_rarities = RarityRanker.rank_collection(collection=test_collection) 103 | expected_tokens_in_rank_order = [ 104 | {"id": 3, "unique_traits": 2, "rank": 1}, 105 | {"id": 0, "unique_traits": 1, "rank": 2}, 106 | {"id": 2, "unique_traits": 1, "rank": 2}, 107 | {"id": 1, "unique_traits": 0, "rank": 4}, 108 | ] 109 | verify_token_rarities(token_rarities, expected_tokens_in_rank_order) 110 | 111 | def test_rarity_ranker_unique_scores(self) -> None: 112 | test_collection = generate_collection_with_token_traits( 113 | [ 114 | # Token 0 115 | { 116 | "trait1": "value1", # 100% 117 | "trait2": "value1", # unique trait 118 | "trait3": "value2", # 75% 119 | }, 120 | # Token 1 121 | { 122 | "trait1": "value1", # 75% 123 | "trait2": "value2", # 50% 124 | "trait3": "value2", # 50% 125 | }, 126 | # Token 2 127 | { 128 | "trait1": "value1", # 100% 129 | "trait2": "value2", # 50% 130 | "trait3": "value3", # unique trait 131 | }, 132 | # Token 3 133 | { 134 | "trait1": "value1", # 100% 135 | "trait2": "value4", # unique trait 136 | "trait3": "value2", # 50% 137 | "trait4": "value1", # unique trait 138 | }, # Token 3 139 | ] 140 | ) 141 | 142 | token_rarities = RarityRanker.rank_collection(test_collection) 143 | scorer = Scorer() 144 | expected_scores = scorer.score_collection(test_collection) 145 | expected_tokens_in_rank_order = [ 146 | { 147 | "id": 3, 148 | "unique_traits": 3, 149 | "rank": 1, 150 | "score": expected_scores[3], 151 | }, 152 | { 153 | "id": 2, 154 | "unique_traits": 1, 155 | "rank": 2, 156 | "score": expected_scores[2], 157 | }, 158 | { 159 | "id": 0, 160 | "unique_traits": 1, 161 | "rank": 3, 162 | "score": expected_scores[0], 163 | }, 164 | { 165 | "id": 1, 166 | "unique_traits": 0, 167 | "rank": 4, 168 | "score": expected_scores[1], 169 | }, 170 | ] 171 | verify_token_rarities(token_rarities, expected_tokens_in_rank_order) 172 | 173 | def test_rarity_ranker_same_scores(self) -> None: 174 | test_collection: Collection = generate_collection_with_token_traits( 175 | [ 176 | { 177 | "trait1": "value1", 178 | "trait2": "value1", 179 | "trait3": "value1", 180 | }, # 0 181 | { 182 | "trait1": "value1", 183 | "trait2": "value1", 184 | "trait3": "value1", 185 | }, # 1 186 | { 187 | "trait1": "value2", 188 | "trait2": "value1", 189 | "trait3": "value3", 190 | }, # 2 191 | { 192 | "trait1": "value2", 193 | "trait2": "value2", 194 | "trait3": "value3", 195 | }, # 3 196 | { 197 | "trait1": "value3", 198 | "trait2": "value3", 199 | "trait3": "value3", 200 | }, # 4 201 | ] 202 | ) 203 | 204 | token_rarities = RarityRanker.rank_collection(test_collection) 205 | 206 | expected_tokens_in_rank_order = [ 207 | {"id": 4, "unique_traits": 2, "rank": 1, "score": 1.3926137488801251}, 208 | {"id": 3, "unique_traits": 1, "rank": 2, "score": 1.1338031424711967}, 209 | {"id": 0, "unique_traits": 0, "rank": 3, "score": 0.8749925360622679}, 210 | {"id": 1, "unique_traits": 0, "rank": 3, "score": 0.8749925360622679}, 211 | {"id": 2, "unique_traits": 0, "rank": 5, "score": 0.7235980365241422}, 212 | ] 213 | verify_token_rarities(token_rarities, expected_tokens_in_rank_order) 214 | 215 | def test_set_ranks_same_unique_different_ic_score(self): 216 | token_rarities: list[TokenRarity] = [] 217 | metadata = TokenMetadata({}, {}, {}) 218 | 219 | token_rarities.append( 220 | TokenRarity( 221 | token=Token( 222 | token_identifier=EVMContractTokenIdentifier( 223 | token_id=1, contract_address="null" 224 | ), 225 | token_standard="ERC-721", 226 | metadata=metadata, 227 | ), 228 | score=1.5, 229 | token_features=TokenRankingFeatures(unique_attribute_count=1), 230 | ) 231 | ) 232 | 233 | token_rarities.append( 234 | TokenRarity( 235 | token=Token( 236 | token_identifier=EVMContractTokenIdentifier( 237 | token_id=2, contract_address="null" 238 | ), 239 | token_standard="ERC-721", 240 | metadata=metadata, 241 | ), 242 | score=1.5, 243 | token_features=TokenRankingFeatures(unique_attribute_count=2), 244 | ) 245 | ) 246 | 247 | token_rarities.append( 248 | TokenRarity( 249 | token=Token( 250 | token_identifier=EVMContractTokenIdentifier( 251 | token_id=3, contract_address="null" 252 | ), 253 | token_standard="ERC-721", 254 | metadata=metadata, 255 | ), 256 | score=0.2, 257 | token_features=TokenRankingFeatures(unique_attribute_count=3), 258 | ) 259 | ) 260 | 261 | token_rarities.append( 262 | TokenRarity( 263 | token=Token( 264 | token_identifier=EVMContractTokenIdentifier( 265 | token_id=4, contract_address="null" 266 | ), 267 | token_standard="ERC-721", 268 | metadata=metadata, 269 | ), 270 | score=7.0, 271 | token_features=TokenRankingFeatures(unique_attribute_count=0), 272 | ) 273 | ) 274 | 275 | result = RarityRanker.set_rarity_ranks(token_rarities) 276 | 277 | assert result[0].token.token_identifier.token_id == 3 278 | assert result[0].rank == 1 279 | 280 | assert result[1].token.token_identifier.token_id == 2 281 | assert result[1].rank == 2 282 | 283 | assert result[2].token.token_identifier.token_id == 1 284 | assert result[2].rank == 2 285 | 286 | assert result[3].token.token_identifier.token_id == 4 287 | assert result[3].rank == 4 288 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /tests/helpers.py: -------------------------------------------------------------------------------- 1 | from random import shuffle 2 | 3 | from open_rarity.models.collection import Collection 4 | from open_rarity.models.token import Token 5 | from open_rarity.models.token_identifier import ( 6 | EVMContractTokenIdentifier, 7 | SolanaMintAddressTokenIdentifier, 8 | ) 9 | from open_rarity.models.token_metadata import ( 10 | AttributeName, 11 | NumericAttribute, 12 | StringAttribute, 13 | TokenMetadata, 14 | ) 15 | from open_rarity.models.token_standard import TokenStandard 16 | 17 | 18 | def create_evm_token( 19 | token_id: int, 20 | contract_address: str = "0xaaa", 21 | token_standard: TokenStandard = TokenStandard.ERC721, 22 | metadata: TokenMetadata | None = None, 23 | ) -> Token: 24 | metadata = metadata or TokenMetadata() 25 | return Token( 26 | token_identifier=EVMContractTokenIdentifier( 27 | contract_address=contract_address, token_id=token_id 28 | ), 29 | token_standard=token_standard, 30 | metadata=metadata, 31 | ) 32 | 33 | 34 | def create_string_evm_token( 35 | token_id: int, 36 | contract_address: str = "0xaaa", 37 | token_standard: TokenStandard = TokenStandard.ERC721, 38 | ) -> Token: 39 | string_metadata = TokenMetadata( 40 | string_attributes={"test name": StringAttribute("test name", "test value")} 41 | ) 42 | return create_evm_token( 43 | token_id=token_id, 44 | contract_address=contract_address, 45 | token_standard=token_standard, 46 | metadata=string_metadata, 47 | ) 48 | 49 | 50 | def create_numeric_evm_token( 51 | token_id: int, 52 | contract_address: str = "0xaaa", 53 | token_standard: TokenStandard = TokenStandard.ERC721, 54 | ) -> Token: 55 | numeric_metadata = TokenMetadata( 56 | numeric_attributes={"test": NumericAttribute("test", 1)} 57 | ) 58 | return create_evm_token( 59 | token_id=token_id, 60 | contract_address=contract_address, 61 | token_standard=token_standard, 62 | metadata=numeric_metadata, 63 | ) 64 | 65 | 66 | def uniform_rarity_tokens( 67 | attribute_count: int = 5, 68 | values_per_attribute: int = 10, 69 | token_total_supply: int = 10000, 70 | ): 71 | tokens = [] 72 | for token_id in range(token_total_supply): 73 | string_attribute_dict = {} 74 | 75 | # Construct attributes for the token such that the first bucket 76 | # gets the first possible value for every attribute, second bucket 77 | # gets the second possible value for every attribute, etc. 78 | for attr_name in range(attribute_count): 79 | string_attribute_dict[str(attr_name)] = StringAttribute( 80 | name=str(attr_name), 81 | value=str(token_id // (token_total_supply // values_per_attribute)), 82 | ) 83 | 84 | tokens.append( 85 | Token( 86 | token_identifier=EVMContractTokenIdentifier( 87 | contract_address="0x0", token_id=token_id 88 | ), 89 | token_standard=TokenStandard.ERC721, 90 | metadata=TokenMetadata(string_attributes=string_attribute_dict), 91 | ) 92 | ) 93 | return tokens 94 | 95 | 96 | def generate_uniform_rarity_collection( 97 | attribute_count: int = 5, 98 | values_per_attribute: int = 10, 99 | token_total_supply: int = 10000, 100 | ) -> Collection: 101 | """generate a Collection with uniformly distributed attributes 102 | where each token gets attributes. 103 | 104 | Every bucket of (token_total_supply // values_per_attribute) gets the same 105 | attributes. 106 | """ 107 | tokens = uniform_rarity_tokens( 108 | token_total_supply=token_total_supply, 109 | attribute_count=attribute_count, 110 | values_per_attribute=values_per_attribute, 111 | ) 112 | 113 | return Collection(name="Uniform Rarity Collection", tokens=tokens) 114 | 115 | 116 | def onerare_rarity_tokens( 117 | attribute_count: int = 3, 118 | values_per_attribute: int = 10, 119 | token_total_supply: int = 10000, 120 | ) -> Collection: 121 | """generate a Collection with a single token with one rare attribute; 122 | otherwise uniformly distributed attributes. 123 | 124 | For default params: 125 | - every bundle of 1111 tokens have exactly the same attributes 126 | - the first 9,999 tokens all have attributes with the same probabilities 127 | - the last token has all unique attributes 128 | Collection attributes frequency: 129 | { 130 | '0': { 131 | '-1': 1111, '0': 1111, '1': 1111, '2': 1111, '3': 1111, 132 | '4': 1111, '5': 1111, '6': 1111, '7': 1111, '9': 1 133 | }, 134 | '1': { 135 | '-1': 1111, '0': 1111, '1': 1111, '2': 1111, '3': 1111, 136 | '4': 1111, '5': 1111, '6': 1111, '7': 1111, '9': 1 137 | }, 138 | '2': { 139 | '-1': 1111, '0': 1111, '1': 1111, '2': 1111, '3': 1111, 140 | '4': 1111, '5': 1111, '6': 1111, '7': 1111, '9': 1 141 | }, 142 | 'meta_trait:trait_count': {'3': 10000} 143 | } 144 | """ 145 | tokens = [] 146 | 147 | # Create attributes for all the uniform tokens 148 | for token_id in range(token_total_supply - 1): 149 | string_attribute_dict = {} 150 | 151 | for attr_name in range(attribute_count): 152 | string_attribute_dict[AttributeName(attr_name)] = StringAttribute( 153 | name=AttributeName(attr_name), 154 | value=str( 155 | token_id // (token_total_supply // (values_per_attribute - 1)) - 1 156 | ), 157 | ) 158 | 159 | tokens.append( 160 | Token( 161 | token_identifier=EVMContractTokenIdentifier( 162 | contract_address="0x0", token_id=token_id 163 | ), 164 | token_standard=TokenStandard.ERC721, 165 | metadata=TokenMetadata(string_attributes=string_attribute_dict), 166 | ) 167 | ) 168 | 169 | # Create the attributes for the last rare token 170 | rare_token_string_attribute_dict = {} 171 | for attr_name in range(attribute_count): 172 | rare_token_string_attribute_dict[AttributeName(attr_name)] = StringAttribute( 173 | name=AttributeName(attr_name), 174 | value=str(values_per_attribute), 175 | ) 176 | 177 | tokens.append( 178 | Token( 179 | token_identifier=EVMContractTokenIdentifier( 180 | contract_address="0x0", token_id=token_total_supply - 1 181 | ), 182 | token_standard=TokenStandard.ERC721, 183 | metadata=TokenMetadata(string_attributes=rare_token_string_attribute_dict), 184 | ) 185 | ) 186 | 187 | return tokens 188 | 189 | 190 | def generate_onerare_rarity_collection( 191 | attribute_count: int = 3, 192 | values_per_attribute: int = 10, 193 | token_total_supply: int = 10000, 194 | ) -> Collection: 195 | """generate a Collection with a single token with one rare attribute; 196 | otherwise uniformly distributed attributes. 197 | 198 | For default params: 199 | - every bundle of 1111 tokens have exactly the same attributes 200 | - the first 9,999 tokens all have attributes with the same probabilities 201 | - the last token has all unique attributes 202 | Collection attributes frequency: 203 | { 204 | '0': { 205 | '-1': 1111, '0': 1111, '1': 1111, '2': 1111, '3': 1111, 206 | '4': 1111, '5': 1111, '6': 1111, '7': 1111, '9': 1 207 | }, 208 | '1': { 209 | '-1': 1111, '0': 1111, '1': 1111, '2': 1111, '3': 1111, 210 | '4': 1111, '5': 1111, '6': 1111, '7': 1111, '9': 1 211 | }, 212 | '2': { 213 | '-1': 1111, '0': 1111, '1': 1111, '2': 1111, '3': 1111, 214 | '4': 1111, '5': 1111, '6': 1111, '7': 1111, '9': 1 215 | }, 216 | 'meta_trait:trait_count': {'3': 10000} 217 | } 218 | """ 219 | tokens = onerare_rarity_tokens( 220 | token_total_supply=token_total_supply, 221 | attribute_count=attribute_count, 222 | values_per_attribute=values_per_attribute, 223 | ) 224 | 225 | return Collection(name="One Rare Rarity Collection", tokens=tokens) 226 | 227 | 228 | def generate_collection_with_token_traits( 229 | tokens_traits: list[dict[str, str | int]], 230 | token_identifier_type: str = "evm_contract", 231 | ) -> Collection: 232 | tokens = [] 233 | for idx, token_traits in enumerate(tokens_traits): 234 | match token_identifier_type: 235 | case EVMContractTokenIdentifier.identifier_type: 236 | identifier_type = EVMContractTokenIdentifier( 237 | contract_address="0x0", token_id=idx 238 | ) 239 | token_standard = TokenStandard.ERC721 240 | case SolanaMintAddressTokenIdentifier.identifier_type: 241 | identifier_type = SolanaMintAddressTokenIdentifier( 242 | mint_address=f"Fake-Address-{idx}" 243 | ) 244 | token_standard = TokenStandard.METAPLEX_NON_FUNGIBLE 245 | case _: 246 | raise ValueError( 247 | f"Unexpected token identifier type: {token_identifier_type}" 248 | ) 249 | 250 | tokens.append( 251 | Token( 252 | token_identifier=identifier_type, 253 | token_standard=token_standard, 254 | metadata=TokenMetadata.from_attributes(token_traits), 255 | ) 256 | ) 257 | 258 | return Collection(name="My collection", tokens=tokens) 259 | 260 | 261 | def get_mixed_trait_spread( 262 | max_total_supply: int = 10000, 263 | ) -> dict[str, dict[str, float]]: 264 | # dict[attribute name, dict[attribute value, items with that attribute]] 265 | return { 266 | "hat": { 267 | "cap": int(max_total_supply * 0.2), 268 | "beanie": int(max_total_supply * 0.3), 269 | "hood": int(max_total_supply * 0.45), 270 | "visor": int(max_total_supply * 0.05), 271 | }, 272 | "shirt": { 273 | "white-t": int(max_total_supply * 0.8), 274 | "vest": int(max_total_supply * 0.2), 275 | }, 276 | "special": { 277 | "true": int(max_total_supply * 0.1), 278 | "null": int(max_total_supply * 0.9), 279 | }, 280 | } 281 | 282 | 283 | def generate_mixed_collection(max_total_supply: int = 10000): 284 | """Generates a collection such that the tokens have traits with 285 | get_mixed_trait_spread() spread of trait occurrences: 286 | "hat": 287 | 20% have "cap", 288 | 30% have "beanie", 289 | 45% have "hood", 290 | 5% have "visor" 291 | "shirt": 292 | 80% have "white-t", 293 | 20% have "vest" 294 | "special": 295 | 1% have "special" 296 | others "null" 297 | Note: The token ids are shuffled and it is random order in terms of 298 | which trait/value combo they get. 299 | """ 300 | if max_total_supply % 10 != 0 or max_total_supply < 100: 301 | raise Exception("only multiples of 10 and greater than 100 please.") 302 | 303 | token_ids = list(range(max_total_supply)) 304 | shuffle(token_ids) 305 | 306 | def get_trait_value(trait_spread, idx): 307 | trait_value_idx = 0 308 | max_idx_for_trait_value = trait_spread[trait_value_idx][1] 309 | while idx >= max_idx_for_trait_value: 310 | trait_value_idx += 1 311 | max_idx_for_trait_value += trait_spread[trait_value_idx][1] 312 | return trait_spread[trait_value_idx][0] 313 | 314 | token_ids_to_traits = {} 315 | for idx, token_id in enumerate(token_ids): 316 | traits = { 317 | trait_name: get_trait_value(list(trait_value_to_percent.items()), idx) 318 | for trait_name, trait_value_to_percent in get_mixed_trait_spread().items() 319 | } 320 | 321 | token_ids_to_traits[token_id] = traits 322 | 323 | return generate_collection_with_token_traits( 324 | [token_ids_to_traits[token_id] for token_id in range(max_total_supply)] 325 | ) 326 | -------------------------------------------------------------------------------- /open_rarity/models/collection.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | from collections import defaultdict 3 | from dataclasses import dataclass 4 | from functools import cached_property 5 | 6 | from open_rarity.models.token import Token 7 | from open_rarity.models.token_metadata import ( 8 | AttributeName, 9 | AttributeValue, 10 | StringAttribute, 11 | ) 12 | from open_rarity.models.token_standard import TokenStandard 13 | from open_rarity.models.utils.attribute_utils import normalize_attribute_string 14 | 15 | TRAIT_COUNT_ATTRIBUTE_NAME = "meta_trait:trait_count" 16 | 17 | 18 | @dataclass 19 | class CollectionAttribute: 20 | """Class represents an attribute that at least one token in a Collection has. 21 | E.g. "hat" = "cap" would be one attribute, and "hat" = "beanie" would be another 22 | unique attribute, even though they may belong to the same attribute type (id=name). 23 | 24 | Attributes 25 | ---------- 26 | attribute : StringAttribute 27 | the unique attribute pair 28 | total_tokens : int 29 | total number of tokens in the collection that have this attribute 30 | """ 31 | 32 | attribute: StringAttribute 33 | total_tokens: int 34 | 35 | 36 | @dataclass 37 | class Collection: 38 | """Class represents collection of tokens used to determine token rarity score. 39 | A token's rarity is influenced by the attribute frequency of all the tokens 40 | in a collection. 41 | 42 | Attributes 43 | ---------- 44 | tokens : list[Token] 45 | list of all Tokens that belong to the collection 46 | attributes_frequency_counts: dict[AttributeName, dict[AttributeValue, int]] 47 | dictionary of attributes to the number of tokens in this collection that has 48 | a specific value for every possible value for the given attribute. 49 | 50 | If not provided, the attributes distribution will be derived from the 51 | attributes on the tokens provided. 52 | 53 | Example: 54 | {"hair": {"brown": 500, "blonde": 100} 55 | which means 500 tokens has hair=brown, 100 token has hair=blonde 56 | 57 | Note: All trait names and string values should be lowercased and stripped 58 | of leading and trialing whitespace. 59 | Note 2: We currently only support string attributes. 60 | 61 | name: A reference string only used for debugger log lines 62 | 63 | We do not recommend resetting @tokens attribute after Collection initialization 64 | as that will mess up cached property values: 65 | has_numeric_attributes 66 | get_token_standards 67 | """ 68 | 69 | attributes_frequency_counts: dict[AttributeName, dict[AttributeValue, int]] 70 | name: str 71 | 72 | def __init__( 73 | self, 74 | tokens: list[Token], 75 | # Deprecated - Kept to not break interface, but is not used. 76 | # We always coimpute the attributes_frequency_counts from the tokens to avoid 77 | # divergence. 78 | # TODO [10/16/22]: To remove in 1.0 release 79 | attributes_frequency_counts: dict[AttributeName, dict[AttributeValue, int]] 80 | | None = None, 81 | name: str | None = "", 82 | ): 83 | if attributes_frequency_counts is not None: 84 | warnings.warn( 85 | "`attribute_frequency_counts` is deprecated and will be removed. " 86 | "Counts will be derived from the token data.", 87 | DeprecationWarning, 88 | stacklevel=2, 89 | ) 90 | self._trait_countify(tokens) 91 | self._tokens = tokens 92 | self.name = name or "" 93 | self.attributes_frequency_counts = ( 94 | self._derive_normalized_attributes_frequency_counts() 95 | ) 96 | 97 | @property 98 | def tokens(self) -> list[Token]: 99 | return self._tokens 100 | 101 | @property 102 | def token_total_supply(self) -> int: 103 | return len(self._tokens) 104 | 105 | @cached_property 106 | def has_numeric_attribute(self) -> bool: 107 | return ( 108 | next( 109 | filter( 110 | lambda t: len(t.metadata.numeric_attributes) 111 | or len(t.metadata.date_attributes), 112 | self._tokens, 113 | ), 114 | None, 115 | ) 116 | is not None 117 | ) 118 | 119 | @cached_property 120 | def token_standards(self) -> list[TokenStandard]: 121 | """Returns token standards for this collection. 122 | 123 | Returns 124 | ------- 125 | list[TokenStandard] 126 | the set of unique token standards that any token in this collection 127 | interfaces or uses. 128 | """ 129 | token_standards = set() 130 | for token in self._tokens: 131 | token_standards.add(token.token_standard) 132 | return list(token_standards) 133 | 134 | def total_tokens_with_attribute(self, attribute: StringAttribute) -> int: 135 | """Returns the numbers of tokens in this collection with the attribute 136 | based on the attributes frequency counts. 137 | 138 | Returns 139 | ------- 140 | int 141 | The number of tokens with attribute (attribute_name, attribute_value) 142 | """ 143 | return self.attributes_frequency_counts.get(attribute.name, {}).get( 144 | attribute.value, 0 145 | ) 146 | 147 | def total_attribute_values(self, attribute_name: str) -> int: 148 | return len(self.attributes_frequency_counts.get(attribute_name, {})) 149 | 150 | def extract_null_attributes( 151 | self, 152 | ) -> dict[AttributeName, CollectionAttribute]: 153 | """Compute probabilities of Null attributes. 154 | 155 | Returns 156 | ------- 157 | dict[AttributeName(str), CollectionAttribute(str)] 158 | dict of attribute name to the number of assets without the attribute 159 | (e.g. # of assets where AttributeName=NULL) 160 | """ 161 | result = {} 162 | for ( 163 | trait_name, 164 | trait_values, 165 | ) in self.attributes_frequency_counts.items(): 166 | # To obtain probabilities for missing attributes 167 | # e.g. value of trait not set for the asset 168 | # 169 | # We sum all values counts for particular 170 | # attributes and subtract it from total supply. 171 | # This number divided by total supply is a 172 | # probability of Null attribute 173 | total_trait_count = sum(trait_values.values()) 174 | 175 | # compute null trait probability 176 | # only if there is a positive number of assets without 177 | # this trait 178 | assets_without_trait = self.token_total_supply - total_trait_count 179 | if assets_without_trait > 0: 180 | result[trait_name] = CollectionAttribute( 181 | attribute=StringAttribute(trait_name, "Null"), 182 | total_tokens=assets_without_trait, 183 | ) 184 | 185 | return result 186 | 187 | def extract_collection_attributes( 188 | self, 189 | ) -> dict[AttributeName, list[CollectionAttribute]]: 190 | """Extracts the map of collection traits with it's respective counts 191 | 192 | Returns 193 | ------- 194 | dict[str, CollectionAttribute] 195 | dict of attribute name to count of assets missing the attribute 196 | """ 197 | 198 | collection_traits: dict[str, list[CollectionAttribute]] = defaultdict(list) 199 | 200 | for ( 201 | trait_name, 202 | trait_value_dict, 203 | ) in self.attributes_frequency_counts.items(): 204 | for trait_value, trait_count in trait_value_dict.items(): 205 | collection_traits[trait_name].append( 206 | CollectionAttribute( 207 | attribute=StringAttribute(trait_name, str(trait_value)), 208 | total_tokens=trait_count, 209 | ) 210 | ) 211 | 212 | return collection_traits 213 | 214 | def _trait_countify(self, tokens: list[Token]) -> None: 215 | """Updates tokens to have meta attribute "meta trait: trait_count" if it doesn't 216 | already exist. 217 | 218 | Parameters 219 | ---------- 220 | tokens : list[Token] 221 | List of tokens to add trait count attribute to. Modifies in place. 222 | 223 | """ 224 | for token in tokens: 225 | trait_count = token.trait_count() 226 | if token.has_attribute(TRAIT_COUNT_ATTRIBUTE_NAME): 227 | trait_count -= 1 228 | # NOTE: There is a chance we override an existing attribute here, but it's 229 | # highly unlikely that a token would have a trait_count attribute to begin 230 | # with (no known collections have it right now). 231 | # To decrease the chance of collision, we pre-pend "meta trait: ". 232 | # If an existing trait count attribute already exists with a different name, 233 | # we will not remove it. In the future, we can refactor to distinguish 234 | # between meta and non-meta attributes. 235 | token.metadata.add_attribute( 236 | StringAttribute(name=TRAIT_COUNT_ATTRIBUTE_NAME, value=str(trait_count)) 237 | ) 238 | 239 | def _normalize_attributes_frequency_counts( 240 | self, 241 | attributes_frequency_counts: dict[AttributeName, dict[AttributeValue, int]], 242 | ) -> dict[AttributeName, dict[AttributeValue, int]]: 243 | """We normalize all collection attributes to ensure that neither casing nor 244 | leading/trailing spaces produce different attributes: 245 | (e.g. 'Hat' == 'hat' == 'hat ') 246 | If a collection has the following in their attributes frequency counts: 247 | ('Hat', 'beanie') 5 tokens and 248 | ('hat', 'beanie') 10 tokens 249 | this would produce: ('hat', 'beanie') 15 tokens 250 | """ 251 | normalized: dict[AttributeName, dict[AttributeValue, int]] = {} 252 | for ( 253 | attr_name, 254 | attr_value_to_count, 255 | ) in attributes_frequency_counts.items(): 256 | normalized_name = normalize_attribute_string(attr_name) 257 | if normalized_name not in normalized: 258 | normalized[normalized_name] = {} 259 | for attr_value, attr_count in attr_value_to_count.items(): 260 | normalized_value = ( 261 | normalize_attribute_string(attr_value) 262 | if isinstance(attr_value, str) 263 | else attr_value 264 | ) 265 | if normalized_value not in normalized[normalized_name]: 266 | normalized[normalized_name][normalized_value] = attr_count 267 | else: 268 | normalized[normalized_name][normalized_value] += attr_count 269 | 270 | return normalized 271 | 272 | def _derive_normalized_attributes_frequency_counts( 273 | self, 274 | ) -> dict[AttributeName, dict[AttributeValue, int]]: 275 | """Derives and constructs attributes_frequency_counts based on 276 | string attributes on tokens. Numeric or date attributes currently not 277 | supported. 278 | 279 | Returns 280 | ------- 281 | dict[ AttributeName, dict[AttributeValue, int] ] 282 | dictionary of attributes to the number of tokens in this collection 283 | that has a specific value for every possible value for the given 284 | attribute, by default None. 285 | """ 286 | attrs_freq_counts: dict[AttributeName, dict[AttributeValue, int]] = defaultdict( 287 | dict 288 | ) 289 | 290 | for token in self._tokens: 291 | for ( 292 | attr_name, 293 | str_attr, 294 | ) in token.metadata.string_attributes.items(): 295 | normalized_name = normalize_attribute_string(attr_name) 296 | if str_attr.value not in attrs_freq_counts[attr_name]: 297 | attrs_freq_counts[normalized_name][str_attr.value] = 1 298 | else: 299 | attrs_freq_counts[normalized_name][str_attr.value] += 1 300 | 301 | return dict(attrs_freq_counts) 302 | 303 | def __str__(self) -> str: 304 | return f"Collection[{self.name}]" 305 | --------------------------------------------------------------------------------