├── .env.sample ├── .github └── workflows │ ├── pull-request.yaml │ └── py-publish.yaml ├── .gitignore ├── .pylintrc ├── COPYRIGHT.md ├── LICENSE ├── Makefile ├── README.md ├── dune_client ├── __init__.py ├── api │ ├── __init__.py │ ├── base.py │ ├── custom.py │ ├── execution.py │ ├── extensions.py │ ├── query.py │ └── table.py ├── client.py ├── client_async.py ├── file │ ├── __init__.py │ ├── base.py │ └── interface.py ├── interface.py ├── models.py ├── py.typed ├── query.py ├── types.py ├── util.py └── viz │ ├── __init__.py │ └── graphs.py ├── pyproject.toml ├── requirements ├── dev.txt └── prod.txt ├── setup.cfg ├── setup.py └── tests ├── e2e ├── test_async_client.py ├── test_client.py └── test_custom_endpoints.py ├── fixtures ├── sample_table_insert.csv └── sample_table_insert.json └── unit ├── test_file.py ├── test_models.py ├── test_query.py ├── test_types.py ├── test_utils.py └── test_viz_sankey.py /.env.sample: -------------------------------------------------------------------------------- 1 | # Required 2 | DUNE_API_KEY= 3 | 4 | # Optional (defaults provided here) 5 | DUNE_API_BASE_URL=https://api.dune.com 6 | DUNE_API_REQUEST_TIMEOUT=10 7 | -------------------------------------------------------------------------------- /.github/workflows/pull-request.yaml: -------------------------------------------------------------------------------- 1 | name: pull request 2 | on: 3 | pull_request: 4 | push: 5 | branches: [ main ] 6 | jobs: 7 | lint-format-types-unit: 8 | runs-on: ubuntu-latest 9 | strategy: 10 | matrix: 11 | python-version: ["3.8", "3.13"] 12 | steps: 13 | - uses: actions/checkout@v4 14 | - name: Set up Python ${{ matrix.python-version }} 15 | uses: actions/setup-python@v5 16 | with: 17 | python-version: ${{ matrix.python-version }} 18 | 19 | - name: Install Requirements 20 | run: pip install -r requirements/dev.txt 21 | - name: Pylint 22 | run: pylint dune_client/ 23 | - name: Black 24 | run: black --check ./ 25 | - name: Type Check (mypy) 26 | run: mypy dune_client --strict 27 | - name: Unit Tests 28 | run: python -m pytest tests/unit 29 | 30 | e2e-tests: 31 | runs-on: ubuntu-latest 32 | steps: 33 | - uses: actions/checkout@v4 34 | - name: Set up Python 3.12 35 | uses: actions/setup-python@v5 36 | with: 37 | python-version: 3.12 38 | 39 | - name: Install Requirements 40 | run: 41 | pip install -r requirements/dev.txt 42 | - name: End to End Tests 43 | env: 44 | DUNE_API_KEY: ${{ secrets.DUNE_API_KEY }} 45 | run: 46 | python -m pytest tests/e2e -------------------------------------------------------------------------------- /.github/workflows/py-publish.yaml: -------------------------------------------------------------------------------- 1 | # This workflow will upload a Python Package using Twine when a release is created 2 | # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries 3 | 4 | # This workflow uses actions that are not certified by GitHub. 5 | # They are provided by a third-party and are governed by 6 | # separate terms of service, privacy policy, and support 7 | # documentation. 8 | 9 | name: Upload Python Package 10 | 11 | on: 12 | release: 13 | types: [published] 14 | 15 | permissions: 16 | contents: read 17 | 18 | jobs: 19 | deploy: 20 | 21 | runs-on: ubuntu-latest 22 | environment: release 23 | permissions: 24 | id-token: write 25 | 26 | steps: 27 | - uses: actions/checkout@v4 28 | - name: Set up Python 29 | uses: actions/setup-python@v5 30 | with: 31 | python-version: 3.9 32 | - name: Install dependencies 33 | run: | 34 | python -m pip install --upgrade pip 35 | pip install build 36 | - name: Build package 37 | run: python -m build 38 | - name: Publish a Python distribution to PyPI 39 | uses: pypa/gh-action-pypi-publish@release/v1 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | __pycache__/ 3 | dist 4 | *.egg-info 5 | _version.py 6 | .idea/ 7 | venv/ 8 | tmp/ 9 | .vscode/ 10 | build/ 11 | .DS_Store -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | disable=fixme,logging-fstring-interpolation,too-many-positional-arguments 3 | [DESIGN] 4 | max-args=10 5 | -------------------------------------------------------------------------------- /COPYRIGHT.md: -------------------------------------------------------------------------------- 1 | # Intellectual Property Notice 2 | 3 | Copyright (c) 2022 Cow Services Lda 4 | Copyright (c) 2023 Dune Analytics AS 5 | 6 | Except as otherwise noted (below and/or in individual files), this project is licensed under 7 | the Apache License, Version 2.0 ([`LICENSE-APACHE`](LICENSE-APACHE) or ). 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | VENV = .venv 2 | PYTHON = $(VENV)/bin/python3 3 | PIP = $(VENV)/bin/pip 4 | 5 | 6 | $(VENV)/bin/activate: requirements/dev.txt 7 | python3 -m venv $(VENV) 8 | $(PIP) install --upgrade pip 9 | $(PIP) install -r requirements/dev.txt 10 | 11 | 12 | install: $(VENV)/bin/activate 13 | 14 | clean: 15 | rm -rf __pycache__ 16 | 17 | fmt: 18 | black ./ 19 | 20 | lint: 21 | pylint dune_client/ 22 | 23 | types: 24 | mypy dune_client/ --strict 25 | 26 | check: fmt lint types 27 | 28 | test-unit: 29 | python -m pytest tests/unit 30 | 31 | test-e2e: 32 | python -m pytest tests/e2e 33 | 34 | test-all: test-unit test-e2e 35 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Python 3.11](https://img.shields.io/badge/python-3.11-blue.svg)](https://www.python.org/downloads/release/python-3102/) 2 | [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) 3 | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 4 | [![Build](https://github.com/duneanalytics/dune-client/actions/workflows/pull-request.yaml/badge.svg)](https://github.com/duneanalytics/dune-client/actions/workflows/pull-request.yaml) 5 | 6 | # Dune Client 7 | 8 | A python framework for interacting with Dune Analytics' [officially supported API 9 | service](https://docs.dune.com/api-reference/overview/introduction). 10 | 11 | ## Installation 12 | 13 | Import as a project dependency 14 | 15 | ```shell 16 | pip install dune-client 17 | ``` 18 | 19 | # Example Usage 20 | 21 | ## Quickstart: run_query 22 | 23 | Export your `DUNE_API_KEY` (or place it in a `.env` file - as in 24 | here [.env.sample](./.env.sample) and `source .env`). 25 | 26 | ```python 27 | from dune_client.types import QueryParameter 28 | from dune_client.client import DuneClient 29 | from dune_client.query import QueryBase 30 | 31 | query = QueryBase( 32 | name="Sample Query", 33 | query_id=1215383, 34 | params=[ 35 | QueryParameter.text_type(name="TextField", value="Word"), 36 | QueryParameter.number_type(name="NumberField", value=3.1415926535), 37 | QueryParameter.date_type(name="DateField", value="2022-05-04 00:00:00"), 38 | QueryParameter.enum_type(name="ListField", value="Option 1"), 39 | ], 40 | ) 41 | print("Results available at", query.url()) 42 | 43 | dune = DuneClient.from_env() 44 | results = dune.run_query(query) 45 | 46 | # or as CSV 47 | # results_csv = dune.run_query_csv(query) 48 | 49 | # or as Pandas Dataframe 50 | # results_df = dune.run_query_dataframe(query) 51 | ``` 52 | 53 | ## Further Examples 54 | 55 | ### Get Latest Results 56 | Use `get_latest_results` to get the most recent query results without using execution credits. 57 | You can specify a `max_age_hours` to re-run the query if the data is too outdated. 58 | 59 | ```python 60 | from dune_client.client import DuneClient 61 | 62 | dune = DuneClient.from_env() 63 | results = dune.get_latest_result(1215383, max_age_hours=8) 64 | ``` 65 | 66 | ## Paid Subscription Features 67 | 68 | ### CRUD Operations 69 | 70 | If you're writing scripts that rely on Dune query results and want to ensure that your local, 71 | peer-reviewed, queries are being used at runtime, you can call `update_query` before `run_query`! 72 | 73 | Here is a fictitious example making use of this functionality; 74 | 75 | ```python 76 | from dune_client.types import QueryParameter 77 | from dune_client.client import DuneClient 78 | 79 | sql = """ 80 | SELECT block_time, hash, 81 | FROM ethereum.transactions 82 | ORDER BY CAST(gas_used as uint256) * CAST(gas_price AS uint256) DESC 83 | LIMIT {{N}} 84 | """ 85 | 86 | dune = DuneClient.from_env() 87 | query = dune.create_query( 88 | name="Top {N} Most Expensive Transactions on Ethereum", 89 | query_sql=sql, 90 | # Optional fields 91 | params=[QueryParameter.number_type(name="N", value=10)], 92 | is_private=False # default 93 | ) 94 | query_id = query.base.query_id 95 | print(f"Created query with id {query.base.query_id}") 96 | # Could retrieve using 97 | # dune.get_query(query_id) 98 | 99 | dune.update_query( 100 | query_id, 101 | # All parameters below are optional 102 | name="Top {N} Most Expensive Transactions on {Blockchain}", 103 | query_sql=sql.replace("ethereum", "{{Blockchain}}"), 104 | params=query.base.parameters() + [QueryParameter.text_type("Blockchain", "ethereum")], 105 | description="Shows time and hash of the most expensive transactions", 106 | tags=["XP€N$IV $H1T"] 107 | ) 108 | 109 | dune.archive_query(query_id) 110 | dune.unarchive_query(query_id) 111 | 112 | dune.make_private(query_id) 113 | dune.make_public(query_id) 114 | ``` 115 | 116 | # Developer Usage & Deployment 117 | 118 | ## Makefile 119 | This project's makefile comes equipped with sufficient commands for local development. 120 | 121 | ### Installation 122 | 123 | ```shell 124 | make install 125 | ```` 126 | 127 | ### Format, Lint & Types 128 | ```shell 129 | make check 130 | ``` 131 | can also be run individually with `fmt`, `lint` and `types` respectively. 132 | 133 | ### Testing 134 | ```shell 135 | make test-unit # Unit tests 136 | make test-e2e # Requires valid `DUNE_API_KEY` 137 | ``` 138 | can also run both with `make test-all` 139 | 140 | ## Deployment 141 | 142 | Publishing releases to PyPi is configured automatically via github actions 143 | (cf. [./.github/workflows/py-publish.yaml](./.github/workflows/py-publish.yaml)). 144 | Any time a branch is tagged for release this workflow is triggered and published with the same version name. 145 | -------------------------------------------------------------------------------- /dune_client/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/duneanalytics/dune-client/1d466e80cd9fe2280d6638e43b94b49067a3817f/dune_client/__init__.py -------------------------------------------------------------------------------- /dune_client/api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/duneanalytics/dune-client/1d466e80cd9fe2280d6638e43b94b49067a3817f/dune_client/api/__init__.py -------------------------------------------------------------------------------- /dune_client/api/base.py: -------------------------------------------------------------------------------- 1 | """ " 2 | Basic Dune Client Class responsible for refreshing Dune Queries 3 | Framework built on Dune's API Documentation 4 | https://docs.dune.com/api-reference/overview/introduction 5 | """ 6 | 7 | from __future__ import annotations 8 | 9 | import logging.config 10 | import os 11 | from json import JSONDecodeError 12 | from typing import Any, Dict, List, Optional, Union, IO 13 | 14 | from requests import Response, Session 15 | from requests.adapters import HTTPAdapter, Retry 16 | 17 | from dune_client.util import get_package_version 18 | 19 | # Headers used for pagination in CSV results 20 | DUNE_CSV_NEXT_URI_HEADER = "x-dune-next-uri" 21 | DUNE_CSV_NEXT_OFFSET_HEADER = "x-dune-next-offset" 22 | # Default maximum number of rows to retrieve per batch of results 23 | MAX_NUM_ROWS_PER_BATCH = 32_000 24 | 25 | 26 | # pylint: disable=too-few-public-methods 27 | class BaseDuneClient: 28 | """ 29 | A Base Client for Dune which sets up default values 30 | and provides some convenient functions to use in other clients 31 | """ 32 | 33 | def __init__( # pylint: disable=too-many-arguments 34 | self, 35 | api_key: str, 36 | base_url: str = "https://api.dune.com", 37 | request_timeout: float = 10, 38 | client_version: str = "v1", 39 | performance: str = "medium", 40 | ): 41 | self.token = api_key 42 | self.base_url = base_url 43 | self.request_timeout = request_timeout 44 | self.client_version = client_version 45 | self.performance = performance 46 | self.logger = logging.getLogger(__name__) 47 | logging.basicConfig(format="%(asctime)s %(levelname)s %(name)s %(message)s") 48 | retry_strategy = Retry( 49 | total=5, 50 | backoff_factor=0.5, 51 | status_forcelist={429, 502, 503, 504}, 52 | allowed_methods={"GET", "POST", "PATCH"}, 53 | raise_on_status=True, 54 | ) 55 | adapter = HTTPAdapter(max_retries=retry_strategy) 56 | self.http = Session() 57 | self.http.mount("https://", adapter) 58 | self.http.mount("http://", adapter) 59 | 60 | @classmethod 61 | def from_env(cls) -> BaseDuneClient: 62 | """ 63 | Constructor allowing user to instantiate a client from environment variable 64 | without having to import dotenv or os manually 65 | We use `DUNE_API_KEY` as the environment variable that holds the API key. 66 | """ 67 | return cls( 68 | api_key=os.environ["DUNE_API_KEY"], 69 | base_url=os.environ.get("DUNE_API_BASE_URL", "https://api.dune.com"), 70 | request_timeout=float(os.environ.get("DUNE_API_REQUEST_TIMEOUT", 10)), 71 | ) 72 | 73 | @property 74 | def api_version(self) -> str: 75 | """Returns client version string""" 76 | return f"/api/{self.client_version}" 77 | 78 | def default_headers(self) -> Dict[str, str]: 79 | """Return default headers containing Dune Api token""" 80 | client_version = get_package_version("dune-client") or "1.3.0" 81 | return { 82 | "x-dune-api-key": self.token, 83 | "User-Agent": f"dune-client/{client_version} (https://pypi.org/project/dune-client/)", 84 | } 85 | 86 | ############ 87 | # Utilities: 88 | ############ 89 | 90 | def _build_parameters( 91 | self, 92 | params: Optional[Dict[str, Union[str, int]]] = None, 93 | columns: Optional[List[str]] = None, 94 | sample_count: Optional[int] = None, 95 | filters: Optional[str] = None, 96 | sort_by: Optional[List[str]] = None, 97 | limit: Optional[int] = None, 98 | offset: Optional[int] = None, 99 | allow_partial_results: str = "true", 100 | ) -> Dict[str, Union[str, int]]: 101 | """ 102 | Utility function that builds a dictionary of parameters to be used 103 | when retrieving advanced results (filters, pagination, sorting, etc.). 104 | This is shared between the sync and async client. 105 | """ 106 | # Ensure we don't specify parameters that are incompatible: 107 | assert ( 108 | # We are not sampling 109 | sample_count is None 110 | # We are sampling and don't use filters or pagination 111 | or (limit is None and offset is None and filters is None) 112 | ), "sampling cannot be combined with filters or pagination" 113 | 114 | params = params or {} 115 | params["allow_partial_results"] = allow_partial_results 116 | if columns is not None and len(columns) > 0: 117 | params["columns"] = ",".join(columns) 118 | if sample_count is not None: 119 | params["sample_count"] = sample_count 120 | if filters is not None: 121 | params["filters"] = filters 122 | if sort_by is not None and len(sort_by) > 0: 123 | params["sort_by"] = ",".join(sort_by) 124 | if limit is not None: 125 | params["limit"] = limit 126 | if offset is not None: 127 | params["offset"] = offset 128 | 129 | return params 130 | 131 | 132 | class BaseRouter(BaseDuneClient): 133 | """Extending the Base Client with elementary api routing""" 134 | 135 | def _handle_response(self, response: Response) -> Any: 136 | """Generic response handler utilized by all Dune API routes""" 137 | try: 138 | # Some responses can be decoded and converted to DuneErrors 139 | response_json = response.json() 140 | self.logger.debug(f"received response {response_json}") 141 | return response_json 142 | except JSONDecodeError as err: 143 | # Others can't. Only raise HTTP error for not decodable errors 144 | response.raise_for_status() 145 | raise ValueError("Unreachable since previous line raises") from err 146 | 147 | def _route_url(self, route: Optional[str] = None, url: Optional[str] = None) -> str: 148 | if route is not None: 149 | final_url = f"{self.base_url}{self.api_version}{route}" 150 | elif url is not None: 151 | final_url = url 152 | else: 153 | assert route is not None or url is not None 154 | 155 | return final_url 156 | 157 | def _get( 158 | self, 159 | route: Optional[str] = None, 160 | params: Optional[Any] = None, 161 | raw: bool = False, 162 | url: Optional[str] = None, 163 | ) -> Any: 164 | """Generic interface for the GET method of a Dune API request""" 165 | final_url = self._route_url(route=route, url=url) 166 | self.logger.debug(f"GET received input url={final_url}") 167 | 168 | response = self.http.get( 169 | url=final_url, 170 | headers=self.default_headers(), 171 | timeout=self.request_timeout, 172 | params=params, 173 | ) 174 | if raw: 175 | return response 176 | return self._handle_response(response) 177 | 178 | def _post( 179 | self, 180 | route: str, 181 | params: Optional[Any] = None, 182 | data: Optional[IO[bytes]] = None, 183 | headers: Optional[Dict[str, str]] = None, 184 | ) -> Any: 185 | """Generic interface for the POST method of a Dune API request""" 186 | url = self._route_url(route) 187 | self.logger.debug(f"POST received input url={url}, params={params}") 188 | response = self.http.post( 189 | url=url, 190 | json=params, 191 | headers=dict(self.default_headers(), **headers if headers else {}), 192 | timeout=self.request_timeout, 193 | data=data, 194 | ) 195 | return self._handle_response(response) 196 | 197 | def _patch(self, route: str, params: Any) -> Any: 198 | """Generic interface for the PATCH method of a Dune API request""" 199 | url = self._route_url(route) 200 | self.logger.debug(f"PATCH received input url={url}, params={params}") 201 | response = self.http.patch( 202 | url=url, 203 | json=params, 204 | headers=self.default_headers(), 205 | timeout=self.request_timeout, 206 | ) 207 | return self._handle_response(response) 208 | 209 | def _delete(self, route: str) -> Any: 210 | """Generic interface for the DELETE method of a Dune API request""" 211 | url = self._route_url(route) 212 | self.logger.debug(f"DELETE received input url={url}") 213 | response = self.http.delete( 214 | url=url, 215 | headers=self.default_headers(), 216 | timeout=self.request_timeout, 217 | ) 218 | return self._handle_response(response) 219 | -------------------------------------------------------------------------------- /dune_client/api/custom.py: -------------------------------------------------------------------------------- 1 | """ 2 | Custom endpoints API enables users to 3 | fetch and filter data from custom endpoints. 4 | """ 5 | 6 | from __future__ import annotations 7 | from typing import List, Optional 8 | 9 | from dune_client.api.base import BaseRouter 10 | from dune_client.models import ( 11 | DuneError, 12 | ResultsResponse, 13 | ) 14 | 15 | 16 | # pylint: disable=duplicate-code 17 | class CustomEndpointAPI(BaseRouter): 18 | """ 19 | Custom endpoints API implementation. 20 | Methods: 21 | get_custom_endpoint_result(): returns the results of a custom endpoint. 22 | """ 23 | 24 | def get_custom_endpoint_result( 25 | self, 26 | handle: str, 27 | endpoint: str, 28 | limit: Optional[int] = None, 29 | offset: Optional[int] = None, 30 | columns: Optional[List[str]] = None, 31 | sample_count: Optional[int] = None, 32 | filters: Optional[str] = None, 33 | sort_by: Optional[List[str]] = None, 34 | ) -> ResultsResponse: 35 | """ 36 | Custom endpoints allow you to fetch and filter data from any 37 | custom endpoint you created. 38 | More information on Custom Endpoints can be round here: 39 | https://docs.dune.com/api-reference/custom/overview 40 | 41 | Args: 42 | handle (str): The handle of the team/user. 43 | endpoint (str): The slug of the custom endpoint. 44 | limit (int, optional): The maximum number of results to return. 45 | offset (int, optional): The number of results to skip. 46 | columns (List[str], optional): A list of columns to return. 47 | sample_count (int, optional): The number of results to return. 48 | filters (str, optional): The filters to apply. 49 | sort_by (List[str], optional): The columns to sort by. 50 | """ 51 | params = self._build_parameters( 52 | columns=columns, 53 | sample_count=sample_count, 54 | filters=filters, 55 | sort_by=sort_by, 56 | limit=limit, 57 | offset=offset, 58 | ) 59 | response_json = self._get( 60 | route=f"/endpoints/{handle}/{endpoint}/results", 61 | params=params, 62 | ) 63 | try: 64 | return ResultsResponse.from_dict(response_json) 65 | except KeyError as err: 66 | raise DuneError(response_json, "ResultsResponse", err) from err 67 | -------------------------------------------------------------------------------- /dune_client/api/execution.py: -------------------------------------------------------------------------------- 1 | """ 2 | Implementation of all Dune API query execution and get results routes. 3 | 4 | Further Documentation: 5 | execution: https://docs.dune.com/api-reference/executions/endpoint/execute-query 6 | get results: https://docs.dune.com/api-reference/executions/endpoint/get-execution-result 7 | """ 8 | 9 | from io import BytesIO 10 | from typing import Any, Dict, List, Optional 11 | 12 | from deprecated import deprecated 13 | 14 | from dune_client.api.base import ( 15 | BaseRouter, 16 | DUNE_CSV_NEXT_URI_HEADER, 17 | DUNE_CSV_NEXT_OFFSET_HEADER, 18 | ) 19 | from dune_client.models import ( 20 | ExecutionResponse, 21 | ExecutionStatusResponse, 22 | ResultsResponse, 23 | ExecutionResultCSV, 24 | DuneError, 25 | ExecutionState, 26 | ) 27 | from dune_client.query import QueryBase 28 | 29 | 30 | class ExecutionAPI(BaseRouter): 31 | """ 32 | Query execution and result fetching functions. 33 | """ 34 | 35 | def execute_query( 36 | self, query: QueryBase, performance: Optional[str] = None 37 | ) -> ExecutionResponse: 38 | """Post's to Dune API for execute `query`""" 39 | params = query.request_format() 40 | params["performance"] = performance or self.performance 41 | 42 | self.logger.info( 43 | f"executing {query.query_id} on {performance or self.performance} cluster" 44 | ) 45 | response_json = self._post( 46 | route=f"/query/{query.query_id}/execute", 47 | params=params, 48 | ) 49 | try: 50 | return ExecutionResponse.from_dict(response_json) 51 | except KeyError as err: 52 | raise DuneError(response_json, "ExecutionResponse", err) from err 53 | 54 | def cancel_execution(self, job_id: str) -> bool: 55 | """POST Execution Cancellation to Dune API for `job_id` (aka `execution_id`)""" 56 | response_json = self._post( 57 | route=f"/execution/{job_id}/cancel", 58 | params=None, 59 | ) 60 | try: 61 | # No need to make a dataclass for this since it's just a boolean. 62 | success: bool = response_json["success"] 63 | return success 64 | except KeyError as err: 65 | raise DuneError(response_json, "CancellationResponse", err) from err 66 | 67 | def get_execution_status(self, job_id: str) -> ExecutionStatusResponse: 68 | """GET status from Dune API for `job_id` (aka `execution_id`)""" 69 | response_json = self._get(route=f"/execution/{job_id}/status") 70 | try: 71 | return ExecutionStatusResponse.from_dict(response_json) 72 | except KeyError as err: 73 | raise DuneError(response_json, "ExecutionStatusResponse", err) from err 74 | 75 | def get_execution_results( 76 | self, 77 | job_id: str, 78 | limit: Optional[int] = None, 79 | offset: Optional[int] = None, 80 | columns: Optional[List[str]] = None, 81 | sample_count: Optional[int] = None, 82 | filters: Optional[str] = None, 83 | sort_by: Optional[List[str]] = None, 84 | allow_partial_results: str = "true", 85 | ) -> ResultsResponse: 86 | """GET results from Dune API for `job_id` (aka `execution_id`)""" 87 | params = self._build_parameters( 88 | columns=columns, 89 | sample_count=sample_count, 90 | filters=filters, 91 | sort_by=sort_by, 92 | limit=limit, 93 | offset=offset, 94 | allow_partial_results=allow_partial_results, 95 | ) 96 | 97 | route = f"/execution/{job_id}/results" 98 | url = self._route_url(route) 99 | return self._get_execution_results_by_url(url=url, params=params) 100 | 101 | def get_execution_results_csv( 102 | self, 103 | job_id: str, 104 | limit: Optional[int] = None, 105 | offset: Optional[int] = None, 106 | columns: Optional[List[str]] = None, 107 | filters: Optional[str] = None, 108 | sample_count: Optional[int] = None, 109 | sort_by: Optional[List[str]] = None, 110 | ) -> ExecutionResultCSV: 111 | """ 112 | GET results in CSV format from Dune API for `job_id` (aka `execution_id`) 113 | 114 | this API only returns the raw data in CSV format, it is faster & lighterweight 115 | use this method for large results where you want lower CPU and memory overhead 116 | if you need metadata information use get_results() or get_status() 117 | """ 118 | params = self._build_parameters( 119 | columns=columns, 120 | sample_count=sample_count, 121 | filters=filters, 122 | sort_by=sort_by, 123 | limit=limit, 124 | offset=offset, 125 | ) 126 | 127 | route = f"/execution/{job_id}/results/csv" 128 | url = self._route_url(route) 129 | return self._get_execution_results_csv_by_url(url=url, params=params) 130 | 131 | def _get_execution_results_by_url( 132 | self, url: str, params: Optional[Dict[str, Any]] = None 133 | ) -> ResultsResponse: 134 | """ 135 | GET results from Dune API with a given URL. This is particularly useful for pagination. 136 | """ 137 | assert url.startswith(self.base_url) 138 | 139 | response_json = self._get(url=url, params=params) 140 | try: 141 | result = ResultsResponse.from_dict(response_json) 142 | if result.state == ExecutionState.PARTIAL: 143 | self.logger.warning( 144 | f"execution {result.execution_id} resulted in a partial " 145 | f"result set (i.e. results too large)." 146 | ) 147 | return result 148 | except KeyError as err: 149 | raise DuneError(response_json, "ResultsResponse", err) from err 150 | 151 | def _get_execution_results_csv_by_url( 152 | self, 153 | url: str, 154 | params: Optional[Dict[str, Any]] = None, 155 | ) -> ExecutionResultCSV: 156 | """ 157 | GET results in CSV format from Dune API with a given URL. This is particularly 158 | useful for pagination 159 | 160 | this API only returns the raw data in CSV format, it is faster & lighterweight 161 | use this method for large results where you want lower CPU and memory overhead 162 | if you need metadata information use get_results() or get_status() 163 | """ 164 | assert url.startswith(self.base_url) 165 | 166 | response = self._get(url=url, params=params, raw=True) 167 | response.raise_for_status() 168 | next_uri = response.headers.get(DUNE_CSV_NEXT_URI_HEADER) 169 | next_offset = response.headers.get(DUNE_CSV_NEXT_OFFSET_HEADER) 170 | return ExecutionResultCSV( 171 | data=BytesIO(response.content), 172 | next_uri=next_uri, 173 | next_offset=next_offset, 174 | ) 175 | 176 | ####################### 177 | # Deprecated Functions: 178 | ####################### 179 | @deprecated(version="1.2.1", reason="Please use execute_query") 180 | def execute( 181 | self, query: QueryBase, performance: Optional[str] = None 182 | ) -> ExecutionResponse: 183 | """Post's to Dune API for execute `query`""" 184 | return self.execute_query(query, performance) 185 | 186 | @deprecated(version="1.2.1", reason="Please use get_execution_status") 187 | def get_status(self, job_id: str) -> ExecutionStatusResponse: 188 | """GET status from Dune API for `job_id` (aka `execution_id`)""" 189 | return self.get_execution_status(job_id) 190 | 191 | @deprecated(version="1.2.1", reason="Please use get_execution_results") 192 | def get_result(self, job_id: str) -> ResultsResponse: 193 | """GET results from Dune API for `job_id` (aka `execution_id`)""" 194 | return self.get_execution_results(job_id) 195 | 196 | @deprecated(version="1.2.1", reason="Please use get_execution_results_csv") 197 | def get_result_csv(self, job_id: str) -> ExecutionResultCSV: 198 | """ 199 | GET results in CSV format from Dune API for `job_id` (aka `execution_id`) 200 | 201 | this API only returns the raw data in CSV format, it is faster & lighterweight 202 | use this method for large results where you want lower CPU and memory overhead 203 | if you need metadata information use get_results() or get_status() 204 | """ 205 | return self.get_execution_results_csv(job_id) 206 | -------------------------------------------------------------------------------- /dune_client/api/extensions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Extended functionality for the ExecutionAPI 3 | """ 4 | 5 | from __future__ import annotations 6 | 7 | import logging 8 | import time 9 | 10 | from io import BytesIO 11 | from typing import Any, List, Optional, Union 12 | 13 | from deprecated import deprecated 14 | 15 | from dune_client.api.base import ( 16 | DUNE_CSV_NEXT_URI_HEADER, 17 | DUNE_CSV_NEXT_OFFSET_HEADER, 18 | MAX_NUM_ROWS_PER_BATCH, 19 | ) 20 | from dune_client.api.execution import ExecutionAPI 21 | from dune_client.api.query import QueryAPI 22 | from dune_client.api.table import TableAPI 23 | from dune_client.api.custom import CustomEndpointAPI 24 | from dune_client.models import ( 25 | ResultsResponse, 26 | DuneError, 27 | ExecutionState, 28 | QueryFailed, 29 | ExecutionResultCSV, 30 | ) 31 | from dune_client.query import QueryBase, parse_query_object_or_id 32 | from dune_client.types import QueryParameter 33 | from dune_client.util import age_in_hours 34 | 35 | # This is the expiry time on old query results. 36 | THREE_MONTHS_IN_HOURS = 2191 37 | # Seconds between checking execution status 38 | POLL_FREQUENCY_SECONDS = 1 39 | 40 | 41 | class ExtendedAPI(ExecutionAPI, QueryAPI, TableAPI, CustomEndpointAPI): 42 | """ 43 | Provides higher level helper methods for faster 44 | and easier development on top of the base ExecutionAPI. 45 | """ 46 | 47 | def run_query( 48 | self, 49 | query: QueryBase, 50 | ping_frequency: int = POLL_FREQUENCY_SECONDS, 51 | performance: Optional[str] = None, 52 | batch_size: Optional[int] = None, 53 | columns: Optional[List[str]] = None, 54 | sample_count: Optional[int] = None, 55 | filters: Optional[str] = None, 56 | sort_by: Optional[List[str]] = None, 57 | allow_partial_results: str = "true", 58 | ) -> ResultsResponse: 59 | """ 60 | Executes a Dune `query`, waits until execution completes, 61 | fetches and returns the results. 62 | Sleeps `ping_frequency` seconds between each status request. 63 | """ 64 | # Ensure we don't specify parameters that are incompatible: 65 | assert ( 66 | # We are not sampling 67 | sample_count is None 68 | # We are sampling and don't use filters or pagination 69 | or (batch_size is None and filters is None) 70 | ), "sampling cannot be combined with filters or pagination" 71 | 72 | if sample_count is not None: 73 | limit = None 74 | else: 75 | limit = batch_size or MAX_NUM_ROWS_PER_BATCH 76 | 77 | # pylint: disable=duplicate-code 78 | job_id = self._refresh(query, ping_frequency, performance) 79 | return self._fetch_entire_result( 80 | self.get_execution_results( 81 | job_id, 82 | columns=columns, 83 | sample_count=sample_count, 84 | filters=filters, 85 | sort_by=sort_by, 86 | limit=limit, 87 | allow_partial_results=allow_partial_results, 88 | ), 89 | ) 90 | 91 | def run_query_csv( 92 | self, 93 | query: QueryBase, 94 | ping_frequency: int = POLL_FREQUENCY_SECONDS, 95 | performance: Optional[str] = None, 96 | batch_size: Optional[int] = None, 97 | columns: Optional[List[str]] = None, 98 | sample_count: Optional[int] = None, 99 | filters: Optional[str] = None, 100 | sort_by: Optional[List[str]] = None, 101 | ) -> ExecutionResultCSV: 102 | """ 103 | Executes a Dune query, waits till execution completes, 104 | fetches and the results in CSV format 105 | (use it load the data directly in pandas.from_csv() or similar frameworks) 106 | """ 107 | # Ensure we don't specify parameters that are incompatible: 108 | assert ( 109 | # We are not sampling 110 | sample_count is None 111 | # We are sampling and don't use filters or pagination 112 | or (batch_size is None and filters is None) 113 | ), "sampling cannot be combined with filters or pagination" 114 | 115 | if sample_count is not None: 116 | limit = None 117 | else: 118 | limit = batch_size or MAX_NUM_ROWS_PER_BATCH 119 | 120 | # pylint: disable=duplicate-code 121 | job_id = self._refresh(query, ping_frequency, performance) 122 | return self._fetch_entire_result_csv( 123 | self.get_execution_results_csv( 124 | job_id, 125 | columns=columns, 126 | sample_count=sample_count, 127 | filters=filters, 128 | sort_by=sort_by, 129 | limit=limit, 130 | ), 131 | ) 132 | 133 | def run_query_dataframe( 134 | self, 135 | query: QueryBase, 136 | ping_frequency: int = POLL_FREQUENCY_SECONDS, 137 | performance: Optional[str] = None, 138 | batch_size: Optional[int] = None, 139 | columns: Optional[List[str]] = None, 140 | sample_count: Optional[int] = None, 141 | filters: Optional[str] = None, 142 | sort_by: Optional[List[str]] = None, 143 | ) -> Any: 144 | """ 145 | Execute a Dune Query, waits till execution completes, 146 | fetched and returns the result as a Pandas DataFrame 147 | 148 | This is a convenience method that uses run_query_csv() + pandas.read_csv() underneath 149 | """ 150 | try: 151 | import pandas # pylint: disable=import-outside-toplevel 152 | except ImportError as exc: 153 | raise ImportError( 154 | "dependency failure, pandas is required but missing" 155 | ) from exc 156 | data = self.run_query_csv( 157 | query, 158 | ping_frequency, 159 | performance, 160 | batch_size=batch_size, 161 | columns=columns, 162 | sample_count=sample_count, 163 | filters=filters, 164 | sort_by=sort_by, 165 | ).data 166 | return pandas.read_csv(data) 167 | 168 | def get_latest_result( 169 | self, 170 | query: Union[QueryBase, str, int], 171 | max_age_hours: int = THREE_MONTHS_IN_HOURS, 172 | batch_size: Optional[int] = None, 173 | columns: Optional[List[str]] = None, 174 | sample_count: Optional[int] = None, 175 | filters: Optional[str] = None, 176 | sort_by: Optional[List[str]] = None, 177 | ) -> ResultsResponse: 178 | """ 179 | GET the latest results for a query_id without re-executing the query 180 | (doesn't use execution credits) 181 | 182 | :param query: :class:`Query` object OR query id as string or int 183 | :param max_age_hours: re-executes the query if result is older than max_age_hours 184 | https://docs.dune.com/api-reference/executions/endpoint/get-query-result 185 | """ 186 | # Ensure we don't specify parameters that are incompatible: 187 | assert ( 188 | # We are not sampling 189 | sample_count is None 190 | # We are sampling and don't use filters or pagination 191 | or (batch_size is None and filters is None) 192 | ), "sampling cannot be combined with filters or pagination" 193 | 194 | params, query_id = parse_query_object_or_id(query) 195 | 196 | # Only fetch 1 row to get metadata first to determine if the result is fresh enough 197 | if params is None: 198 | params = {} 199 | params["limit"] = 1 200 | 201 | response_json = self._get( 202 | route=f"/query/{query_id}/results", 203 | params=params, 204 | ) 205 | try: 206 | if sample_count is None and batch_size is None: 207 | batch_size = MAX_NUM_ROWS_PER_BATCH 208 | metadata = ResultsResponse.from_dict(response_json) 209 | last_run = metadata.times.execution_ended_at 210 | 211 | if last_run and age_in_hours(last_run) > max_age_hours: 212 | # Query older than specified max age, we need to refresh the results 213 | logging.info( 214 | f"results (from {last_run}) older than {max_age_hours} hours, re-running query" 215 | ) 216 | results = self.run_query( 217 | query if isinstance(query, QueryBase) else QueryBase(query_id), 218 | columns=columns, 219 | sample_count=sample_count, 220 | filters=filters, 221 | sort_by=sort_by, 222 | batch_size=batch_size, 223 | ) 224 | else: 225 | # The results are fresh enough, retrieve the entire result 226 | # pylint: disable=duplicate-code 227 | results = self._fetch_entire_result( 228 | self.get_execution_results( 229 | metadata.execution_id, 230 | columns=columns, 231 | sample_count=sample_count, 232 | filters=filters, 233 | sort_by=sort_by, 234 | limit=batch_size, 235 | ), 236 | ) 237 | return results 238 | except KeyError as err: 239 | raise DuneError(response_json, "ResultsResponse", err) from err 240 | 241 | def get_latest_result_dataframe( 242 | self, 243 | query: Union[QueryBase, str, int], 244 | batch_size: Optional[int] = None, 245 | columns: Optional[List[str]] = None, 246 | sample_count: Optional[int] = None, 247 | filters: Optional[str] = None, 248 | sort_by: Optional[List[str]] = None, 249 | ) -> Any: 250 | """ 251 | GET the latest results for a query_id without re-executing the query 252 | (doesn't use execution credits) 253 | returns the result as a Pandas DataFrame 254 | 255 | This is a convenience method that uses get_latest_result() + pandas.read_csv() underneath 256 | """ 257 | try: 258 | import pandas # pylint: disable=import-outside-toplevel 259 | except ImportError as exc: 260 | raise ImportError( 261 | "dependency failure, pandas is required but missing" 262 | ) from exc 263 | 264 | results = self.download_csv( 265 | query, 266 | columns=columns, 267 | sample_count=sample_count, 268 | filters=filters, 269 | sort_by=sort_by, 270 | batch_size=batch_size, 271 | ) 272 | return pandas.read_csv(results.data) 273 | 274 | def download_csv( 275 | self, 276 | query: Union[QueryBase, str, int], 277 | batch_size: Optional[int] = None, 278 | columns: Optional[List[str]] = None, 279 | sample_count: Optional[int] = None, 280 | filters: Optional[str] = None, 281 | sort_by: Optional[List[str]] = None, 282 | ) -> ExecutionResultCSV: 283 | """ 284 | Almost like an alias for `get_latest_result` but for the csv endpoint. 285 | https://docs.dune.com/api-reference/executions/endpoint/get-query-result-csv 286 | """ 287 | # Ensure we don't specify parameters that are incompatible: 288 | assert ( 289 | # We are not sampling 290 | sample_count is None 291 | # We are sampling and don't use filters or pagination 292 | or (batch_size is None and filters is None) 293 | ), "sampling cannot be combined with filters or pagination" 294 | 295 | params, query_id = parse_query_object_or_id(query) 296 | 297 | params = self._build_parameters( 298 | params=params, 299 | columns=columns, 300 | sample_count=sample_count, 301 | filters=filters, 302 | sort_by=sort_by, 303 | limit=batch_size, 304 | ) 305 | if sample_count is None and batch_size is None: 306 | params["limit"] = MAX_NUM_ROWS_PER_BATCH 307 | 308 | response = self._get( 309 | route=f"/query/{query_id}/results/csv", params=params, raw=True 310 | ) 311 | response.raise_for_status() 312 | 313 | next_uri = response.headers.get(DUNE_CSV_NEXT_URI_HEADER) 314 | next_offset = response.headers.get(DUNE_CSV_NEXT_OFFSET_HEADER) 315 | return self._fetch_entire_result_csv( 316 | ExecutionResultCSV( 317 | data=BytesIO(response.content), 318 | next_uri=next_uri, 319 | next_offset=next_offset, 320 | ), 321 | ) 322 | 323 | ############################################################################################## 324 | # Plus Features: these features use APIs that are only available on paid subscription plans 325 | ############################################################################################## 326 | 327 | def run_sql( 328 | self, 329 | query_sql: str, 330 | params: Optional[list[QueryParameter]] = None, 331 | is_private: bool = True, 332 | archive_after: bool = True, 333 | performance: Optional[str] = None, 334 | ping_frequency: int = POLL_FREQUENCY_SECONDS, 335 | name: str = "API Query", 336 | ) -> ResultsResponse: 337 | """ 338 | Allows user to provide execute raw_sql via the CRUD interface 339 | - create, run, get results with optional archive/delete. 340 | - Query is by default made private and archived after execution. 341 | Requires Plus subscription! 342 | """ 343 | query = self.create_query(name, query_sql, params, is_private) 344 | try: 345 | results = self.run_query( 346 | query=query.base, performance=performance, ping_frequency=ping_frequency 347 | ) 348 | finally: 349 | if archive_after: 350 | self.archive_query(query.base.query_id) 351 | return results 352 | 353 | ###################### 354 | # Deprecated Functions 355 | ###################### 356 | @deprecated(version="1.2.1", reason="Please use run_query") 357 | def refresh( 358 | self, 359 | query: QueryBase, 360 | ping_frequency: int = POLL_FREQUENCY_SECONDS, 361 | performance: Optional[str] = None, 362 | ) -> ResultsResponse: 363 | """ 364 | Executes a Dune `query`, waits until execution completes, 365 | fetches and returns the results. 366 | Sleeps `ping_frequency` seconds between each status request. 367 | """ 368 | return self.run_query(query, ping_frequency, performance) 369 | 370 | @deprecated(version="1.2.1", reason="Please use run_query_csv") 371 | def refresh_csv( 372 | self, 373 | query: QueryBase, 374 | ping_frequency: int = POLL_FREQUENCY_SECONDS, 375 | performance: Optional[str] = None, 376 | ) -> ExecutionResultCSV: 377 | """ 378 | Executes a Dune query, waits till execution completes, 379 | fetches and the results in CSV format 380 | (use it load the data directly in pandas.from_csv() or similar frameworks) 381 | """ 382 | return self.run_query_csv(query, ping_frequency, performance) 383 | 384 | @deprecated(version="1.2.1", reason="Please use run_query_dataframe") 385 | def refresh_into_dataframe( 386 | self, 387 | query: QueryBase, 388 | ping_frequency: int = POLL_FREQUENCY_SECONDS, 389 | performance: Optional[str] = None, 390 | ) -> Any: 391 | """ 392 | Execute a Dune Query, waits till execution completes, 393 | fetched and returns the result as a Pandas DataFrame 394 | 395 | This is a convenience method that uses refresh_csv underneath 396 | """ 397 | return self.run_query_dataframe(query, ping_frequency, performance) 398 | 399 | ################# 400 | # Private Methods 401 | ################# 402 | def _refresh( 403 | self, 404 | query: QueryBase, 405 | ping_frequency: int = POLL_FREQUENCY_SECONDS, 406 | performance: Optional[str] = None, 407 | ) -> str: 408 | """ 409 | Executes a Dune `query`, waits until execution completes, 410 | fetches and returns the results. 411 | Sleeps `ping_frequency` seconds between each status request. 412 | """ 413 | job_id = self.execute_query(query=query, performance=performance).execution_id 414 | status = self.get_execution_status(job_id) 415 | while status.state not in ExecutionState.terminal_states(): 416 | self.logger.info( 417 | f"waiting for query execution {job_id} to complete: {status}" 418 | ) 419 | time.sleep(ping_frequency) 420 | status = self.get_execution_status(job_id) 421 | if status.state == ExecutionState.PENDING: 422 | self.logger.warning("Partial result set retrieved.") 423 | if status.state == ExecutionState.FAILED: 424 | self.logger.error(status) 425 | raise QueryFailed(f"Error data: {status.error}") 426 | return job_id 427 | 428 | def _fetch_entire_result( 429 | self, 430 | results: ResultsResponse, 431 | ) -> ResultsResponse: 432 | """ 433 | Retrieve the entire results using the paginated API 434 | """ 435 | next_uri = results.next_uri 436 | while next_uri is not None: 437 | batch = self._get_execution_results_by_url(url=next_uri) 438 | results += batch 439 | next_uri = batch.next_uri 440 | 441 | return results 442 | 443 | def _fetch_entire_result_csv( 444 | self, 445 | results: ExecutionResultCSV, 446 | ) -> ExecutionResultCSV: 447 | """ 448 | Retrieve the entire results in CSV format using the paginated API 449 | """ 450 | next_uri = results.next_uri 451 | while next_uri is not None: 452 | batch = self._get_execution_results_csv_by_url(url=next_uri) 453 | results += batch 454 | next_uri = batch.next_uri 455 | 456 | return results 457 | -------------------------------------------------------------------------------- /dune_client/api/query.py: -------------------------------------------------------------------------------- 1 | """ 2 | CRUD API endpoints enables users to 3 | create, read, update, make public/private or archive queries beyond the Dune IDE. 4 | Enables more flexible integration of Dune API into your workflow 5 | and freeing you from UI-exclusive query editing. 6 | """ 7 | 8 | from __future__ import annotations 9 | from typing import Optional, Any 10 | 11 | from dune_client.api.base import BaseRouter 12 | from dune_client.models import DuneError 13 | from dune_client.query import DuneQuery 14 | from dune_client.types import QueryParameter 15 | 16 | 17 | class QueryAPI(BaseRouter): 18 | """ 19 | Implementation of Query API (aka CRUD) Operations - Plus subscription only 20 | https://docs.dune.com/api-reference/queries/endpoint/query-object 21 | """ 22 | 23 | def create_query( 24 | self, 25 | name: str, 26 | query_sql: str, 27 | params: Optional[list[QueryParameter]] = None, 28 | is_private: bool = False, 29 | ) -> DuneQuery: 30 | """ 31 | Creates Dune Query by ID 32 | https://docs.dune.com/api-reference/queries/endpoint/create 33 | """ 34 | payload = { 35 | "name": name, 36 | "query_sql": query_sql, 37 | "is_private": is_private, 38 | } 39 | if params is not None: 40 | payload["parameters"] = [p.to_dict() for p in params] 41 | response_json = self._post(route="/query/", params=payload) 42 | try: 43 | query_id = int(response_json["query_id"]) 44 | # Note that this requires an extra request. 45 | return self.get_query(query_id) 46 | except KeyError as err: 47 | raise DuneError(response_json, "CreateQueryResponse", err) from err 48 | 49 | def get_query(self, query_id: int) -> DuneQuery: 50 | """ 51 | Retrieves Dune Query by ID 52 | https://docs.dune.com/api-reference/queries/endpoint/read 53 | """ 54 | response_json = self._get(route=f"/query/{query_id}") 55 | return DuneQuery.from_dict(response_json) 56 | 57 | def update_query( # pylint: disable=too-many-arguments 58 | self, 59 | query_id: int, 60 | name: Optional[str] = None, 61 | query_sql: Optional[str] = None, 62 | params: Optional[list[QueryParameter]] = None, 63 | description: Optional[str] = None, 64 | tags: Optional[list[str]] = None, 65 | ) -> int: 66 | """ 67 | Updates Dune Query by ID 68 | https://docs.dune.com/api-reference/queries/endpoint/update 69 | 70 | The request body should contain all fields that need to be updated. 71 | Any omitted fields will be left untouched. 72 | If the tags or parameters are provided as an empty array, 73 | they will be deleted from the query. 74 | """ 75 | parameters: dict[str, Any] = {} 76 | if name is not None: 77 | parameters["name"] = name 78 | if description is not None: 79 | parameters["description"] = description 80 | if tags is not None: 81 | parameters["tags"] = tags 82 | if query_sql is not None: 83 | parameters["query_sql"] = query_sql 84 | if params is not None: 85 | parameters["parameters"] = [p.to_dict() for p in params] 86 | 87 | if not bool(parameters): 88 | # Nothing to change no need to make reqeust 89 | self.logger.warning("called update_query with no proposed changes.") 90 | return query_id 91 | 92 | response_json = self._patch( 93 | route=f"/query/{query_id}", 94 | params=parameters, 95 | ) 96 | try: 97 | # No need to make a dataclass for this since it's just a boolean. 98 | return int(response_json["query_id"]) 99 | except KeyError as err: 100 | raise DuneError(response_json, "UpdateQueryResponse", err) from err 101 | 102 | def archive_query(self, query_id: int) -> bool: 103 | """ 104 | https://docs.dune.com/api-reference/queries/endpoint/archive 105 | returns resulting value of Query.is_archived 106 | """ 107 | response_json = self._post(route=f"/query/{query_id}/archive") 108 | try: 109 | # No need to make a dataclass for this since it's just a boolean. 110 | return self.get_query(int(response_json["query_id"])).meta.is_archived 111 | except KeyError as err: 112 | raise DuneError(response_json, "ArchiveQueryResponse", err) from err 113 | 114 | def unarchive_query(self, query_id: int) -> bool: 115 | """ 116 | https://docs.dune.com/api-reference/queries/endpoint/unarchive 117 | returns resulting value of Query.is_archived 118 | """ 119 | response_json = self._post(route=f"/query/{query_id}/unarchive") 120 | try: 121 | # No need to make a dataclass for this since it's just a boolean. 122 | return self.get_query(int(response_json["query_id"])).meta.is_archived 123 | except KeyError as err: 124 | raise DuneError(response_json, "UnarchiveQueryResponse", err) from err 125 | 126 | def make_private(self, query_id: int) -> None: 127 | """ 128 | https://docs.dune.com/api-reference/queries/endpoint/private 129 | """ 130 | response_json = self._post(route=f"/query/{query_id}/private") 131 | try: 132 | assert self.get_query(int(response_json["query_id"])).meta.is_private 133 | except KeyError as err: 134 | raise DuneError(response_json, "MakePrivateResponse", err) from err 135 | 136 | def make_public(self, query_id: int) -> None: 137 | """ 138 | https://docs.dune.com/api-reference/queries/endpoint/unprivate 139 | """ 140 | response_json = self._post(route=f"/query/{query_id}/unprivate") 141 | try: 142 | assert not self.get_query(int(response_json["query_id"])).meta.is_private 143 | except KeyError as err: 144 | raise DuneError(response_json, "MakePublicResponse", err) from err 145 | -------------------------------------------------------------------------------- /dune_client/api/table.py: -------------------------------------------------------------------------------- 1 | """ 2 | Table API endpoints enables users to 3 | create and insert data into Dune. 4 | """ 5 | 6 | from __future__ import annotations 7 | from typing import List, Dict, IO 8 | 9 | from dune_client.api.base import BaseRouter 10 | from dune_client.models import ( 11 | DuneError, 12 | InsertTableResult, 13 | CreateTableResult, 14 | DeleteTableResult, 15 | ClearTableResult, 16 | ) 17 | 18 | 19 | class TableAPI(BaseRouter): 20 | """ 21 | Implementation of Table endpoints - Plus subscription only 22 | https://docs.dune.com/api-reference/tables/ 23 | """ 24 | 25 | def upload_csv( 26 | self, 27 | table_name: str, 28 | data: str, 29 | description: str = "", 30 | is_private: bool = False, 31 | ) -> bool: 32 | """ 33 | https://docs.dune.com/api-reference/tables/endpoint/upload 34 | This endpoint allows you to upload any .csv file into Dune. The only limitations are: 35 | 36 | - File has to be < 200 MB 37 | - Column names in the table can't start with a special character or digits. 38 | - Private uploads require a Plus subscription. 39 | 40 | Below are the specifics of how to work with the API. 41 | """ 42 | response_json = self._post( 43 | route="/table/upload/csv", 44 | params={ 45 | "table_name": table_name, 46 | "description": description, 47 | "data": data, 48 | "is_private": is_private, 49 | }, 50 | ) 51 | try: 52 | return bool(response_json["success"]) 53 | except KeyError as err: 54 | raise DuneError(response_json, "UploadCsvResponse", err) from err 55 | 56 | def create_table( 57 | self, 58 | namespace: str, 59 | table_name: str, 60 | schema: List[Dict[str, str]], 61 | description: str = "", 62 | is_private: bool = False, 63 | ) -> CreateTableResult: 64 | """ 65 | https://docs.dune.com/api-reference/tables/endpoint/create 66 | The create table endpoint allows you to create an empty table 67 | with a specific schema in Dune. 68 | 69 | The only limitations are: 70 | - If a table already exists with the same name, the request will fail. 71 | - Column names in the table can’t start with a special character or a digit. 72 | """ 73 | 74 | result_json = self._post( 75 | route="/table/create", 76 | params={ 77 | "namespace": namespace, 78 | "table_name": table_name, 79 | "schema": schema, 80 | "description": description, 81 | "is_private": is_private, 82 | }, 83 | ) 84 | try: 85 | return CreateTableResult.from_dict(result_json) 86 | except KeyError as err: 87 | raise DuneError(result_json, "CreateTableResult", err) from err 88 | 89 | def insert_table( 90 | self, 91 | namespace: str, 92 | table_name: str, 93 | data: IO[bytes], 94 | content_type: str, 95 | ) -> InsertTableResult: 96 | """ 97 | https://docs.dune.com/api-reference/tables/endpoint/insert 98 | The insert table endpoint allows you to insert data into an existing table in Dune. 99 | 100 | The only limitations are: 101 | - The file has to be in json or csv format 102 | - The file has to have the same schema as the table 103 | """ 104 | 105 | result_json = self._post( 106 | route=f"/table/{namespace}/{table_name}/insert", 107 | headers={"Content-Type": content_type}, 108 | data=data, 109 | ) 110 | try: 111 | return InsertTableResult.from_dict(result_json) 112 | except KeyError as err: 113 | raise DuneError(result_json, "InsertTableResult", err) from err 114 | 115 | def clear_data(self, namespace: str, table_name: str) -> ClearTableResult: 116 | """ 117 | https://docs.dune.com/api-reference/tables/endpoint/clear 118 | The Clear endpoint removes all the data in the specified table, 119 | but does not delete the table. 120 | """ 121 | 122 | result_json = self._post(route=f"/table/{namespace}/{table_name}/clear") 123 | try: 124 | return ClearTableResult.from_dict(result_json) 125 | except KeyError as err: 126 | raise DuneError(result_json, "ClearTableResult", err) from err 127 | 128 | def delete_table(self, namespace: str, table_name: str) -> DeleteTableResult: 129 | """ 130 | https://docs.dune.com/api-reference/tables/endpoint/delete 131 | The delete table endpoint allows you to delete an existing table in Dune. 132 | """ 133 | 134 | response_json = self._delete(route=f"/table/{namespace}/{table_name}") 135 | try: 136 | return DeleteTableResult.from_dict(response_json) 137 | except KeyError as err: 138 | raise DuneError(response_json, "DeleteTableResult", err) from err 139 | -------------------------------------------------------------------------------- /dune_client/client.py: -------------------------------------------------------------------------------- 1 | """ " 2 | Basic Dune Client Class responsible for refreshing Dune Queries 3 | Framework built on Dune's API Documentation 4 | https://docs.dune.com/api-reference/overview/introduction 5 | """ 6 | 7 | from dune_client.api.extensions import ExtendedAPI 8 | 9 | 10 | class DuneClient(ExtendedAPI): 11 | """ 12 | An interface for Dune API with a few convenience methods 13 | combining the use of endpoints (e.g. run_query) 14 | 15 | Inheritance Hierarchy sketched as follows: 16 | 17 | DuneClient 18 | | 19 | |--- ExtendedAPI 20 | | - Contains compositions of execution methods like `run_query` 21 | | (things like `run_query`, `run_query_csv`, etc..) 22 | | - make use of both Execution and Query APIs 23 | | 24 | |--- ExecutionAPI(BaseRouter) 25 | | - Contains query execution methods. 26 | | 27 | |--- QueryAPI(BaseRouter) 28 | | - Contains CRUD Operations on Queries 29 | """ 30 | -------------------------------------------------------------------------------- /dune_client/client_async.py: -------------------------------------------------------------------------------- 1 | """ " 2 | Async Dune Client Class responsible for refreshing Dune Queries 3 | Framework built on Dune's API Documentation 4 | https://docs.dune.com/api-reference/overview/introduction 5 | """ 6 | 7 | from __future__ import annotations 8 | 9 | import asyncio 10 | import ssl 11 | from io import BytesIO 12 | from typing import Any, Callable, Dict, List, Optional, Union 13 | 14 | import certifi 15 | from aiohttp import ( 16 | ClientResponseError, 17 | ClientSession, 18 | ClientResponse, 19 | ContentTypeError, 20 | TCPConnector, 21 | ClientTimeout, 22 | ) 23 | 24 | from dune_client.api.base import ( 25 | BaseDuneClient, 26 | DUNE_CSV_NEXT_URI_HEADER, 27 | DUNE_CSV_NEXT_OFFSET_HEADER, 28 | MAX_NUM_ROWS_PER_BATCH, 29 | ) 30 | from dune_client.models import ( 31 | ExecutionResponse, 32 | ExecutionResultCSV, 33 | DuneError, 34 | QueryFailed, 35 | ExecutionStatusResponse, 36 | ResultsResponse, 37 | ExecutionState, 38 | ) 39 | 40 | from dune_client.query import QueryBase, parse_query_object_or_id 41 | 42 | 43 | class RetryableError(Exception): 44 | """ 45 | Internal exception used to signal that the request should be retried 46 | """ 47 | 48 | def __init__(self, base_error: ClientResponseError) -> None: 49 | self.base_error = base_error 50 | 51 | 52 | class MaxRetryError(Exception): 53 | """ 54 | This exception is raised when the maximum number of retries is exceeded, 55 | e.g. due to rate limiting or internal server errors 56 | """ 57 | 58 | def __init__(self, url: str, reason: Exception | None = None) -> None: 59 | self.reason = reason 60 | 61 | message = f"Max retries exceeded with url: {url} (Caused by {reason!r})" 62 | 63 | super().__init__(message) 64 | 65 | 66 | # pylint: disable=duplicate-code 67 | class AsyncDuneClient(BaseDuneClient): 68 | """ 69 | An asynchronous interface for Dune API with a few convenience methods 70 | combining the use of endpoints (e.g. refresh) 71 | """ 72 | 73 | _connection_limit = 3 74 | 75 | def __init__( 76 | self, 77 | api_key: str, 78 | base_url: str = "https://api.dune.com", 79 | request_timeout: float = 10, 80 | client_version: str = "v1", 81 | performance: str = "medium", 82 | connection_limit: int = 3, 83 | ): 84 | """ 85 | api_key - Dune API key 86 | connection_limit - number of parallel requests to execute. 87 | For non-pro accounts Dune allows only up to 3 requests but that number can be increased. 88 | """ 89 | super().__init__( 90 | api_key, base_url, request_timeout, client_version, performance 91 | ) 92 | self._connection_limit = connection_limit 93 | self._session: Optional[ClientSession] = None 94 | 95 | async def _create_session(self) -> ClientSession: 96 | # Create an SSL context using the certifi certificate store 97 | ssl_context = ssl.create_default_context(cafile=certifi.where()) 98 | 99 | conn = TCPConnector(limit=self._connection_limit, ssl=ssl_context) 100 | return ClientSession( 101 | connector=conn, 102 | base_url=self.base_url, 103 | timeout=ClientTimeout(total=self.request_timeout), 104 | ) 105 | 106 | async def connect(self) -> None: 107 | """Opens a client session (can be used instead of async with)""" 108 | self._session = await self._create_session() 109 | 110 | async def disconnect(self) -> None: 111 | """Closes client session""" 112 | if self._session: 113 | await self._session.close() 114 | 115 | async def __aenter__(self) -> AsyncDuneClient: 116 | self._session = await self._create_session() 117 | return self 118 | 119 | async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: 120 | await self.disconnect() 121 | 122 | async def _handle_response(self, response: ClientResponse) -> Any: 123 | if response.status in {429, 502, 503, 504}: 124 | try: 125 | response.raise_for_status() 126 | except ClientResponseError as err: 127 | raise RetryableError( 128 | base_error=err, 129 | ) from err 130 | try: 131 | # Some responses can be decoded and converted to DuneErrors 132 | response_json = await response.json() 133 | self.logger.debug(f"received response {response_json}") 134 | return response_json 135 | except ContentTypeError as err: 136 | # Others can't. Only raise HTTP error for not decodable errors 137 | response.raise_for_status() 138 | raise ValueError("Unreachable since previous line raises") from err 139 | 140 | def _route_url( 141 | self, 142 | route: Optional[str] = None, 143 | url: Optional[str] = None, 144 | ) -> str: 145 | if route is not None: 146 | final_route = f"{self.api_version}{route}" 147 | elif url is not None: 148 | assert url.startswith(self.base_url) 149 | final_route = url[len(self.base_url) :] 150 | else: 151 | assert route is not None or url is not None 152 | 153 | return final_route 154 | 155 | async def _handle_ratelimit(self, call: Callable[..., Any], url: str) -> Any: 156 | """Generic wrapper around request callables. If the request fails due to rate limiting, 157 | or server side errors, it will retry it up to five times, sleeping i * 5s in between 158 | """ 159 | backoff_factor = 0.5 160 | error: Optional[ClientResponseError] = None 161 | for i in range(5): 162 | try: 163 | return await call() 164 | except RetryableError as e: 165 | self.logger.warning( 166 | f"Rate limited or internal error. Retrying in {i * 5} seconds..." 167 | ) 168 | error = e.base_error 169 | await asyncio.sleep(i**2 * backoff_factor) 170 | 171 | raise MaxRetryError(url, error) 172 | 173 | async def _get( 174 | self, 175 | route: Optional[str] = None, 176 | params: Optional[Any] = None, 177 | raw: bool = False, 178 | url: Optional[str] = None, 179 | ) -> Any: 180 | final_route = self._route_url(route=route, url=url) 181 | self.logger.debug(f"GET received input route={final_route}") 182 | 183 | async def _get() -> Any: 184 | if self._session is None: 185 | raise ValueError("Client is not connected; call `await cl.connect()`") 186 | response = await self._session.get( 187 | url=final_route, 188 | headers=self.default_headers(), 189 | params=params, 190 | ) 191 | if raw: 192 | return response 193 | return await self._handle_response(response) 194 | 195 | return await self._handle_ratelimit(_get, final_route) 196 | 197 | async def _post(self, route: str, params: Any) -> Any: 198 | url = self._route_url(route) 199 | self.logger.debug(f"POST received input url={url}, params={params}") 200 | 201 | async def _post() -> Any: 202 | if self._session is None: 203 | raise ValueError("Client is not connected; call `await cl.connect()`") 204 | response = await self._session.post( 205 | url=url, 206 | json=params, 207 | headers=self.default_headers(), 208 | ) 209 | return await self._handle_response(response) 210 | 211 | return await self._handle_ratelimit(_post, route) 212 | 213 | async def execute( 214 | self, query: QueryBase, performance: Optional[str] = None 215 | ) -> ExecutionResponse: 216 | """Post's to Dune API for execute `query`""" 217 | params = query.request_format() 218 | params["performance"] = performance or self.performance 219 | 220 | self.logger.info( 221 | f"executing {query.query_id} on {performance or self.performance} cluster" 222 | ) 223 | response_json = await self._post( 224 | route=f"/query/{query.query_id}/execute", 225 | params=params, 226 | ) 227 | try: 228 | return ExecutionResponse.from_dict(response_json) 229 | except KeyError as err: 230 | raise DuneError(response_json, "ExecutionResponse", err) from err 231 | 232 | async def get_status(self, job_id: str) -> ExecutionStatusResponse: 233 | """GET status from Dune API for `job_id` (aka `execution_id`)""" 234 | response_json = await self._get(route=f"/execution/{job_id}/status") 235 | try: 236 | return ExecutionStatusResponse.from_dict(response_json) 237 | except KeyError as err: 238 | raise DuneError(response_json, "ExecutionStatusResponse", err) from err 239 | 240 | async def get_result( 241 | self, 242 | job_id: str, 243 | batch_size: Optional[int] = None, 244 | columns: Optional[List[str]] = None, 245 | sample_count: Optional[int] = None, 246 | filters: Optional[str] = None, 247 | sort_by: Optional[List[str]] = None, 248 | ) -> ResultsResponse: 249 | """GET results from Dune API for `job_id` (aka `execution_id`)""" 250 | assert ( 251 | # We are not sampling 252 | sample_count is None 253 | # We are sampling and don't use filters or pagination 254 | or (batch_size is None and filters is None) 255 | ), "sampling cannot be combined with filters or pagination" 256 | 257 | if sample_count is None and batch_size is None: 258 | batch_size = MAX_NUM_ROWS_PER_BATCH 259 | 260 | results = await self._get_result_page( 261 | job_id, 262 | columns=columns, 263 | sample_count=sample_count, 264 | filters=filters, 265 | sort_by=sort_by, 266 | limit=batch_size, 267 | ) 268 | while results.next_uri is not None: 269 | batch = await self._get_result_by_url(results.next_uri) 270 | results += batch 271 | 272 | return results 273 | 274 | async def get_result_csv( 275 | self, 276 | job_id: str, 277 | batch_size: Optional[int] = None, 278 | columns: Optional[List[str]] = None, 279 | sample_count: Optional[int] = None, 280 | filters: Optional[str] = None, 281 | sort_by: Optional[List[str]] = None, 282 | ) -> ExecutionResultCSV: 283 | """ 284 | GET results in CSV format from Dune API for `job_id` (aka `execution_id`) 285 | 286 | this API only returns the raw data in CSV format, it is faster & lighterweight 287 | use this method for large results where you want lower CPU and memory overhead 288 | if you need metadata information use get_results() or get_status() 289 | """ 290 | assert ( 291 | # We are not sampling 292 | sample_count is None 293 | # We are sampling and don't use filters or pagination 294 | or (batch_size is None and filters is None) 295 | ), "sampling cannot be combined with filters or pagination" 296 | 297 | if sample_count is None and batch_size is None: 298 | batch_size = MAX_NUM_ROWS_PER_BATCH 299 | 300 | results = await self._get_result_csv_page( 301 | job_id, 302 | columns=columns, 303 | sample_count=sample_count, 304 | filters=filters, 305 | sort_by=sort_by, 306 | limit=batch_size, 307 | ) 308 | while results.next_uri is not None: 309 | batch = await self._get_result_csv_by_url(results.next_uri) 310 | results += batch 311 | 312 | return results 313 | 314 | async def get_latest_result( 315 | self, 316 | query: Union[QueryBase, str, int], 317 | batch_size: int = MAX_NUM_ROWS_PER_BATCH, 318 | ) -> ResultsResponse: 319 | """ 320 | GET the latest results for a query_id without having to execute the query again. 321 | 322 | :param query: :class:`Query` object OR query id as string | int 323 | 324 | https://docs.dune.com/api-reference/executions/endpoint/get-query-result 325 | """ 326 | params, query_id = parse_query_object_or_id(query) 327 | 328 | if params is None: 329 | params = {} 330 | 331 | params["limit"] = batch_size 332 | 333 | response_json = await self._get( 334 | route=f"/query/{query_id}/results", 335 | params=params, 336 | ) 337 | try: 338 | results = ResultsResponse.from_dict(response_json) 339 | while results.next_uri is not None: 340 | batch = await self._get_result_by_url(results.next_uri) 341 | results += batch 342 | 343 | return results 344 | except KeyError as err: 345 | raise DuneError(response_json, "ResultsResponse", err) from err 346 | 347 | async def cancel_execution(self, job_id: str) -> bool: 348 | """POST Execution Cancellation to Dune API for `job_id` (aka `execution_id`)""" 349 | response_json = await self._post( 350 | route=f"/execution/{job_id}/cancel", 351 | params=None, 352 | ) 353 | try: 354 | # No need to make a dataclass for this since it's just a boolean. 355 | success: bool = response_json["success"] 356 | return success 357 | except KeyError as err: 358 | raise DuneError(response_json, "CancellationResponse", err) from err 359 | 360 | ######################## 361 | # Higher level functions 362 | ######################## 363 | 364 | async def refresh( 365 | self, 366 | query: QueryBase, 367 | ping_frequency: int = 5, 368 | performance: Optional[str] = None, 369 | batch_size: Optional[int] = None, 370 | columns: Optional[List[str]] = None, 371 | sample_count: Optional[int] = None, 372 | filters: Optional[str] = None, 373 | sort_by: Optional[List[str]] = None, 374 | ) -> ResultsResponse: 375 | """ 376 | Executes a Dune `query`, waits until execution completes, 377 | fetches and returns the results. 378 | Sleeps `ping_frequency` seconds between each status request. 379 | """ 380 | assert ( 381 | # We are not sampling 382 | sample_count is None 383 | # We are sampling and don't use filters or pagination 384 | or (batch_size is None and filters is None) 385 | ), "sampling cannot be combined with filters or pagination" 386 | 387 | job_id = await self._refresh( 388 | query, ping_frequency=ping_frequency, performance=performance 389 | ) 390 | return await self.get_result( 391 | job_id, 392 | columns=columns, 393 | sample_count=sample_count, 394 | filters=filters, 395 | sort_by=sort_by, 396 | batch_size=batch_size, 397 | ) 398 | 399 | async def refresh_csv( 400 | self, 401 | query: QueryBase, 402 | ping_frequency: int = 5, 403 | performance: Optional[str] = None, 404 | batch_size: Optional[int] = None, 405 | columns: Optional[List[str]] = None, 406 | sample_count: Optional[int] = None, 407 | filters: Optional[str] = None, 408 | sort_by: Optional[List[str]] = None, 409 | ) -> ExecutionResultCSV: 410 | """ 411 | Executes a Dune query, waits till execution completes, 412 | fetches and the results in CSV format 413 | (use it load the data directly in pandas.from_csv() or similar frameworks) 414 | """ 415 | assert ( 416 | # We are not sampling 417 | sample_count is None 418 | # We are sampling and don't use filters or pagination 419 | or (batch_size is None and filters is None) 420 | ), "sampling cannot be combined with filters or pagination" 421 | 422 | job_id = await self._refresh( 423 | query, ping_frequency=ping_frequency, performance=performance 424 | ) 425 | return await self.get_result_csv( 426 | job_id, 427 | columns=columns, 428 | sample_count=sample_count, 429 | filters=filters, 430 | sort_by=sort_by, 431 | batch_size=batch_size, 432 | ) 433 | 434 | async def refresh_into_dataframe( 435 | self, 436 | query: QueryBase, 437 | performance: Optional[str] = None, 438 | batch_size: Optional[int] = None, 439 | columns: Optional[List[str]] = None, 440 | sample_count: Optional[int] = None, 441 | filters: Optional[str] = None, 442 | sort_by: Optional[List[str]] = None, 443 | ) -> Any: 444 | """ 445 | Execute a Dune Query, waits till execution completes, 446 | fetched and returns the result as a Pandas DataFrame 447 | 448 | This is a convenience method that uses refresh_csv underneath 449 | """ 450 | try: 451 | import pandas # pylint: disable=import-outside-toplevel 452 | except ImportError as exc: 453 | raise ImportError( 454 | "dependency failure, pandas is required but missing" 455 | ) from exc 456 | results = await self.refresh_csv( 457 | query, 458 | performance=performance, 459 | columns=columns, 460 | sample_count=sample_count, 461 | filters=filters, 462 | sort_by=sort_by, 463 | batch_size=batch_size, 464 | ) 465 | return pandas.read_csv(results.data) 466 | 467 | ################# 468 | # Private Methods 469 | ################# 470 | 471 | async def _get_result_page( 472 | self, 473 | job_id: str, 474 | limit: Optional[int] = None, 475 | offset: Optional[int] = None, 476 | columns: Optional[List[str]] = None, 477 | sample_count: Optional[int] = None, 478 | filters: Optional[str] = None, 479 | sort_by: Optional[List[str]] = None, 480 | ) -> ResultsResponse: 481 | """GET a page of results from Dune API for `job_id` (aka `execution_id`)""" 482 | 483 | if sample_count is None and limit is None and offset is None: 484 | limit = MAX_NUM_ROWS_PER_BATCH 485 | offset = 0 486 | 487 | params = self._build_parameters( 488 | columns=columns, 489 | sample_count=sample_count, 490 | filters=filters, 491 | sort_by=sort_by, 492 | limit=limit, 493 | offset=offset, 494 | ) 495 | response_json = await self._get( 496 | route=f"/execution/{job_id}/results", 497 | params=params, 498 | ) 499 | 500 | try: 501 | return ResultsResponse.from_dict(response_json) 502 | except KeyError as err: 503 | raise DuneError(response_json, "ResultsResponse", err) from err 504 | 505 | async def _get_result_by_url( 506 | self, 507 | url: str, 508 | params: Optional[Dict[str, Any]] = None, 509 | ) -> ResultsResponse: 510 | """ 511 | GET results from Dune API with a given URL. This is particularly useful for pagination. 512 | """ 513 | response_json = await self._get(url=url, params=params) 514 | 515 | try: 516 | return ResultsResponse.from_dict(response_json) 517 | except KeyError as err: 518 | raise DuneError(response_json, "ResultsResponse", err) from err 519 | 520 | async def _get_result_csv_page( 521 | self, 522 | job_id: str, 523 | limit: Optional[int] = None, 524 | offset: Optional[int] = None, 525 | columns: Optional[List[str]] = None, 526 | sample_count: Optional[int] = None, 527 | filters: Optional[str] = None, 528 | sort_by: Optional[List[str]] = None, 529 | ) -> ExecutionResultCSV: 530 | """ 531 | GET a page of results in CSV format from Dune API for `job_id` (aka `execution_id`) 532 | """ 533 | 534 | if sample_count is None and limit is None and offset is None: 535 | limit = MAX_NUM_ROWS_PER_BATCH 536 | offset = 0 537 | 538 | params = self._build_parameters( 539 | columns=columns, 540 | sample_count=sample_count, 541 | filters=filters, 542 | sort_by=sort_by, 543 | limit=limit, 544 | offset=offset, 545 | ) 546 | 547 | route = f"/execution/{job_id}/results/csv" 548 | response = await self._get(route=route, params=params, raw=True) 549 | response.raise_for_status() 550 | 551 | next_uri = response.headers.get(DUNE_CSV_NEXT_URI_HEADER) 552 | next_offset = response.headers.get(DUNE_CSV_NEXT_OFFSET_HEADER) 553 | return ExecutionResultCSV( 554 | data=BytesIO(await response.content.read(-1)), 555 | next_uri=next_uri, 556 | next_offset=next_offset, 557 | ) 558 | 559 | async def _get_result_csv_by_url( 560 | self, 561 | url: str, 562 | params: Optional[Dict[str, Any]] = None, 563 | ) -> ExecutionResultCSV: 564 | """ 565 | GET results in CSV format from Dune API with a given URL. 566 | This is particularly useful for pagination. 567 | """ 568 | response = await self._get(url=url, params=params, raw=True) 569 | response.raise_for_status() 570 | 571 | next_uri = response.headers.get(DUNE_CSV_NEXT_URI_HEADER) 572 | next_offset = response.headers.get(DUNE_CSV_NEXT_OFFSET_HEADER) 573 | return ExecutionResultCSV( 574 | data=BytesIO(await response.content.read(-1)), 575 | next_uri=next_uri, 576 | next_offset=next_offset, 577 | ) 578 | 579 | async def _refresh( 580 | self, 581 | query: QueryBase, 582 | ping_frequency: int = 5, 583 | performance: Optional[str] = None, 584 | ) -> str: 585 | """ 586 | Executes a Dune `query`, waits until execution completes, 587 | fetches and returns the results. 588 | Sleeps `ping_frequency` seconds between each status request. 589 | """ 590 | job_id = (await self.execute(query=query, performance=performance)).execution_id 591 | status = await self.get_status(job_id) 592 | while status.state not in ExecutionState.terminal_states(): 593 | self.logger.info( 594 | f"waiting for query execution {job_id} to complete: {status}" 595 | ) 596 | await asyncio.sleep(ping_frequency) 597 | status = await self.get_status(job_id) 598 | if status.state == ExecutionState.FAILED: 599 | self.logger.error(status) 600 | raise QueryFailed(f"Error data: {status.error}") 601 | 602 | return job_id 603 | -------------------------------------------------------------------------------- /dune_client/file/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/duneanalytics/dune-client/1d466e80cd9fe2280d6638e43b94b49067a3817f/dune_client/file/__init__.py -------------------------------------------------------------------------------- /dune_client/file/base.py: -------------------------------------------------------------------------------- 1 | """File Reader and Writer for DuneRecords""" 2 | 3 | from __future__ import annotations 4 | 5 | import csv 6 | import json 7 | import logging 8 | import os.path 9 | from abc import ABC, abstractmethod 10 | from pathlib import Path 11 | from typing import TextIO, List, Tuple 12 | 13 | # ndjson missing types: https://github.com/rhgrant10/ndjson/issues/10 14 | import ndjson # type: ignore 15 | 16 | from dune_client.types import DuneRecord 17 | 18 | logger = logging.getLogger(__name__) 19 | logging.basicConfig(format="%(asctime)s %(levelname)s %(name)s %(message)s") 20 | 21 | 22 | class FileRWInterface(ABC): 23 | """Interface for File Read, Write and Append functionality (specific to Dune Query Results)""" 24 | 25 | def __init__(self, path: Path | str, name: str, encoding: str = "utf-8"): 26 | self.path = path 27 | self.filename = name 28 | self.encoding = encoding 29 | 30 | @property 31 | def filepath(self) -> str: 32 | """Internal method for building absolute path.""" 33 | return os.path.join(self.path, self.filename) 34 | 35 | @abstractmethod 36 | def _assert_matching_keys(self, keys: Tuple[str, ...]) -> None: 37 | """Used as validation for append""" 38 | 39 | @abstractmethod 40 | def load(self, file: TextIO) -> list[DuneRecord]: 41 | """Loads DuneRecords from `file`""" 42 | 43 | @abstractmethod 44 | def write( 45 | self, out_file: TextIO, data: list[DuneRecord], skip_headers: bool = False 46 | ) -> None: 47 | """Writes `data` to `out_file`""" 48 | 49 | def append(self, data: List[DuneRecord]) -> None: 50 | """Appends `data` to file with `name`""" 51 | if len(data) > 0: 52 | self._assert_matching_keys(tuple(data[0].keys())) 53 | with open(self.filepath, "a+", encoding=self.encoding) as out_file: 54 | return self.write(out_file, data, skip_headers=True) 55 | 56 | 57 | class CSVFile(FileRWInterface): 58 | """File Read/Writer for CSV format""" 59 | 60 | def _assert_matching_keys(self, keys: Tuple[str, ...]) -> None: 61 | with open(self.filepath, "r", encoding=self.encoding) as file: 62 | # Check matching headers. 63 | headers = file.readline() 64 | existing_keys = headers.strip().split(",") 65 | 66 | key_tuple = tuple(existing_keys) 67 | assert keys == key_tuple, f"{keys} != {key_tuple}" 68 | 69 | def load(self, file: TextIO) -> list[DuneRecord]: 70 | """Loads DuneRecords from `file`""" 71 | return list(csv.DictReader(file)) 72 | 73 | def write( 74 | self, out_file: TextIO, data: list[DuneRecord], skip_headers: bool = False 75 | ) -> None: 76 | """Writes `data` to `out_file`""" 77 | if len(data) == 0: 78 | logger.warning( 79 | "Writing an empty CSV file with headers -- will not work with append later." 80 | ) 81 | return 82 | headers = data[0].keys() 83 | data_tuple = [tuple(rec.values()) for rec in data] 84 | dict_writer = csv.DictWriter(out_file, headers, lineterminator="\n") 85 | if not skip_headers: 86 | dict_writer.writeheader() 87 | writer = csv.writer(out_file, lineterminator="\n") 88 | writer.writerows(data_tuple) 89 | 90 | 91 | class JSONFile(FileRWInterface): 92 | """File Read/Writer for JSON format""" 93 | 94 | def _assert_matching_keys(self, keys: Tuple[str, ...]) -> None: 95 | with open(self.filepath, "r", encoding=self.encoding) as file: 96 | single_object = json.loads(file.readline())[0] 97 | existing_keys = single_object.keys() 98 | 99 | key_tuple = tuple(existing_keys) 100 | assert keys == key_tuple, f"{keys} != {key_tuple}" 101 | 102 | def load(self, file: TextIO) -> list[DuneRecord]: 103 | """Loads DuneRecords from `file`""" 104 | loaded_file: list[DuneRecord] = json.loads(file.read()) 105 | return loaded_file 106 | 107 | def write( 108 | self, out_file: TextIO, data: list[DuneRecord], skip_headers: bool = False 109 | ) -> None: 110 | """Writes `data` to `out_file`""" 111 | out_file.write(json.dumps(data)) 112 | 113 | def append(self, data: List[DuneRecord]) -> None: 114 | """Appends `data` to file with `name`""" 115 | if len(data) > 0: 116 | self._assert_matching_keys(tuple(data[0].keys())) 117 | with open(self.filepath, "r", encoding=self.encoding) as existing_file: 118 | existing_data = self.load(existing_file) 119 | with open(self.filepath, "w", encoding=self.encoding) as existing_file: 120 | self.write(existing_file, existing_data + data) 121 | 122 | 123 | class NDJSONFile(FileRWInterface): 124 | """File Read/Writer for NDJSON format""" 125 | 126 | def _assert_matching_keys(self, keys: Tuple[str, ...]) -> None: 127 | with open(self.filepath, "r", encoding=self.encoding) as file: 128 | single_object = json.loads(file.readline()) 129 | existing_keys = single_object.keys() 130 | 131 | key_tuple = tuple(existing_keys) 132 | assert keys == key_tuple, f"{keys} != {key_tuple}" 133 | 134 | def load(self, file: TextIO) -> list[DuneRecord]: 135 | """Loads DuneRecords from `file`""" 136 | return list(ndjson.reader(file)) 137 | 138 | def write( 139 | self, out_file: TextIO, data: list[DuneRecord], skip_headers: bool = False 140 | ) -> None: 141 | """Writes `data` to `out_file`""" 142 | writer = ndjson.writer(out_file, ensure_ascii=False) 143 | for row in data: 144 | writer.writerow(row) 145 | -------------------------------------------------------------------------------- /dune_client/file/interface.py: -------------------------------------------------------------------------------- 1 | """File Reader and Writer for DuneRecords""" 2 | 3 | from __future__ import annotations 4 | 5 | import logging 6 | import os.path 7 | from os.path import exists 8 | from pathlib import Path 9 | from typing import Callable, List 10 | 11 | from dune_client.file.base import FileRWInterface, CSVFile, JSONFile, NDJSONFile 12 | from dune_client.types import DuneRecord 13 | 14 | logger = logging.getLogger(__name__) 15 | logging.basicConfig(format="%(asctime)s %(levelname)s %(name)s %(message)s") 16 | 17 | 18 | class FileIO: 19 | """ 20 | CSV is a more compact file type, 21 | but requires iteration over the set pre and post write 22 | JSON is a redundant file format 23 | but writes the content exactly as it is received from Dune. 24 | NDJSON used for data "streams" 25 | """ 26 | 27 | def __init__( 28 | self, 29 | path: Path | str, 30 | encoding: str = "utf-8", 31 | ): 32 | if not os.path.exists(path): 33 | logger.info(f"creating write path {path}") 34 | os.makedirs(path) 35 | self.path = path 36 | self.encoding: str = encoding 37 | 38 | def _write( 39 | self, 40 | data: List[DuneRecord], 41 | writer: FileRWInterface, 42 | skip_empty: bool, 43 | ) -> None: 44 | # The following three lines are duplicated in _append, due to python version compatibility 45 | # https://github.com/cowprotocol/dune-client/issues/45 46 | # We will continue to support python < 3.10 until ~3.13, this issue will remain open. 47 | if skip_empty and len(data) == 0: 48 | logger.info(f"Nothing to write to {writer.filename}... skipping") 49 | return None 50 | with open(writer.filepath, "w", encoding=self.encoding) as out_file: 51 | writer.write(out_file, data) 52 | return None 53 | 54 | def _append( 55 | self, 56 | data: List[DuneRecord], 57 | writer: FileRWInterface, 58 | skip_empty: bool, 59 | ) -> None: 60 | fname = writer.filename 61 | if skip_empty and len(data) == 0: 62 | logger.info(f"Nothing to write to {fname}... skipping") 63 | return None 64 | if not exists(writer.filepath): 65 | logger.warning( 66 | f"File {fname} does not exist, using write instead of append!" 67 | ) 68 | return self._write(data, writer, skip_empty) 69 | 70 | return writer.append(data) 71 | 72 | def append_csv( 73 | self, 74 | data: list[DuneRecord], 75 | name: str, 76 | skip_empty: bool = True, 77 | ) -> None: 78 | """Appends `data` to csv file `name`""" 79 | # This is a special case because we want to skip headers when the file already exists 80 | # Additionally, we may want to validate that the headers actually coincide. 81 | self._append(data, CSVFile(self.path, name, self.encoding), skip_empty) 82 | 83 | def append_json( 84 | self, data: list[DuneRecord], name: str, skip_empty: bool = True 85 | ) -> None: 86 | """ 87 | Appends `data` to json file `name` 88 | This is the least efficient of all, since we have to load the entire file, 89 | concatenate the lists and then overwrite the file! 90 | Other filetypes such as CSV and NDJSON can be directly appended to! 91 | """ 92 | self._append(data, JSONFile(self.path, name, self.encoding), skip_empty) 93 | 94 | def append_ndjson( 95 | self, data: list[DuneRecord], name: str, skip_empty: bool = True 96 | ) -> None: 97 | """Appends `data` to ndjson file `name`""" 98 | self._append(data, NDJSONFile(self.path, name, self.encoding), skip_empty) 99 | 100 | def write_csv( 101 | self, data: list[DuneRecord], name: str, skip_empty: bool = True 102 | ) -> None: 103 | """Writes `data` to csv file `name`""" 104 | self._write(data, CSVFile(self.path, name, self.encoding), skip_empty) 105 | 106 | def write_json( 107 | self, data: list[DuneRecord], name: str, skip_empty: bool = True 108 | ) -> None: 109 | """Writes `data` to json file `name`""" 110 | self._write(data, JSONFile(self.path, name, self.encoding), skip_empty) 111 | 112 | def write_ndjson( 113 | self, data: list[DuneRecord], name: str, skip_empty: bool = True 114 | ) -> None: 115 | """Writes `data` to ndjson file `name`""" 116 | self._write(data, NDJSONFile(self.path, name, self.encoding), skip_empty) 117 | 118 | def _load(self, reader: FileRWInterface) -> list[DuneRecord]: 119 | """Loads DuneRecords from file `name`""" 120 | with open(reader.filepath, "r", encoding=self.encoding) as file: 121 | return reader.load(file) 122 | 123 | def load_csv(self, name: str) -> list[DuneRecord]: 124 | """Loads DuneRecords from csv file `name`""" 125 | return self._load(CSVFile(self.path, name, self.encoding)) 126 | 127 | def load_json(self, name: str) -> list[DuneRecord]: 128 | """Loads DuneRecords from json file `name`""" 129 | return self._load(JSONFile(self.path, name, self.encoding)) 130 | 131 | def load_ndjson(self, name: str) -> list[DuneRecord]: 132 | """Loads DuneRecords from ndjson file `name`""" 133 | return self._load(NDJSONFile(self.path, name, self.encoding)) 134 | 135 | def _parse_ftype(self, name: str, ftype: FileRWInterface | str) -> FileRWInterface: 136 | if isinstance(ftype, str): 137 | lowered_value = ftype.lower() 138 | if "ndjson" in lowered_value: 139 | return NDJSONFile(self.path, name, self.encoding) 140 | if "json" in lowered_value: 141 | return JSONFile(self.path, name, self.encoding) 142 | if "csv" in lowered_value: 143 | return CSVFile(self.path, name, self.encoding) 144 | raise ValueError(f"Could not determine file type from {ftype}!") 145 | return ftype 146 | 147 | def load_singleton( 148 | self, name: str, ftype: FileRWInterface | str, index: int = 0 149 | ) -> DuneRecord: 150 | """Loads and returns single entry by index (default 0)""" 151 | reader = self._parse_ftype(name, ftype) 152 | return self._load(reader)[index] 153 | 154 | 155 | WriteLikeSignature = Callable[[FileIO, List[DuneRecord], str, FileRWInterface], None] 156 | -------------------------------------------------------------------------------- /dune_client/interface.py: -------------------------------------------------------------------------------- 1 | """ 2 | Abstract class for a basic Dune Interface with refresh method used by Query Runner. 3 | """ 4 | 5 | import abc 6 | from typing import Any 7 | 8 | from dune_client.models import ResultsResponse, ExecutionResultCSV 9 | from dune_client.query import QueryBase 10 | 11 | 12 | # pylint: disable=too-few-public-methods 13 | class DuneInterface(abc.ABC): 14 | """ 15 | User Facing Methods for a Dune Client 16 | """ 17 | 18 | @abc.abstractmethod 19 | def refresh(self, query: QueryBase) -> ResultsResponse: 20 | """ 21 | Executes a Dune query, waits till query execution completes, 22 | fetches and returns the results. 23 | """ 24 | 25 | @abc.abstractmethod 26 | def refresh_csv(self, query: QueryBase) -> ExecutionResultCSV: 27 | """ 28 | Executes a Dune query, waits till execution completes, 29 | fetches and the results in CSV format 30 | (use it load the data directly in pandas.from_csv() or similar frameworks) 31 | 32 | this Dune API only returns the raw data in CSV format, it is faster & lighterweight 33 | use this method for large results where you want lower CPU and memory overhead 34 | """ 35 | 36 | @abc.abstractmethod 37 | def refresh_into_dataframe(self, query: QueryBase) -> Any: 38 | """ 39 | Execute a Dune Query, waits till execution completes, 40 | fetched and returns the result as a Pandas DataFrame 41 | 42 | This is a convenience method that uses refresh_csv underneath 43 | it assumes the caller has already called `import pandas` 44 | """ 45 | -------------------------------------------------------------------------------- /dune_client/models.py: -------------------------------------------------------------------------------- 1 | """ 2 | Dataclasses encoding response data from Dune API. 3 | """ 4 | 5 | from __future__ import annotations 6 | 7 | import logging.config 8 | from dataclasses import dataclass 9 | from datetime import datetime 10 | from enum import Enum 11 | from io import BytesIO 12 | from os import SEEK_END 13 | from typing import Optional, Any, Union, List, Dict 14 | from dataclasses_json import DataClassJsonMixin 15 | from dateutil.parser import parse 16 | 17 | from dune_client.types import DuneRecord 18 | 19 | log = logging.getLogger(__name__) 20 | logging.basicConfig( 21 | format="%(asctime)s %(levelname)s %(name)s %(message)s", level=logging.INFO 22 | ) 23 | 24 | 25 | class QueryFailed(Exception): 26 | """Special Error for failed Queries""" 27 | 28 | 29 | class DuneError(Exception): 30 | """Possibilities seen so far 31 | {'error': 'invalid API Key'} 32 | {'error': 'Query not found'} 33 | {'error': 'An internal error occured'} 34 | {'error': 'The requested execution ID (ID: Wonky Job ID) is invalid.'} 35 | """ 36 | 37 | def __init__(self, data: dict[str, str], response_class: str, err: KeyError): 38 | error_message = f"Can't build {response_class} from {data}" 39 | log.error(f"{error_message} due to KeyError: {err}") 40 | super().__init__(error_message) 41 | 42 | 43 | class ExecutionState(Enum): 44 | """ 45 | Enum for possible values of Query Execution 46 | """ 47 | 48 | COMPLETED = "QUERY_STATE_COMPLETED" 49 | EXECUTING = "QUERY_STATE_EXECUTING" 50 | PARTIAL = "QUERY_STATE_COMPLETED_PARTIAL" 51 | PENDING = "QUERY_STATE_PENDING" 52 | CANCELLED = "QUERY_STATE_CANCELLED" 53 | FAILED = "QUERY_STATE_FAILED" 54 | EXPIRED = "QUERY_STATE_EXPIRED" 55 | 56 | @classmethod 57 | def terminal_states(cls) -> set[ExecutionState]: 58 | """ 59 | Returns the terminal states (i.e. when a query execution is no longer executing 60 | """ 61 | return {cls.COMPLETED, cls.CANCELLED, cls.FAILED, cls.EXPIRED, cls.PARTIAL} 62 | 63 | def is_complete(self) -> bool: 64 | """Returns True is state is completed, otherwise False.""" 65 | return self == ExecutionState.COMPLETED 66 | 67 | 68 | @dataclass 69 | class ExecutionResponse: 70 | """ 71 | Representation of Response from Dune's [Post] Execute Query ID endpoint 72 | """ 73 | 74 | execution_id: str 75 | state: ExecutionState 76 | 77 | @classmethod 78 | def from_dict(cls, data: dict[str, str]) -> ExecutionResponse: 79 | """Constructor from dictionary. See unit test for sample input.""" 80 | return cls( 81 | execution_id=data["execution_id"], state=ExecutionState(data["state"]) 82 | ) 83 | 84 | 85 | @dataclass 86 | class TimeData: 87 | """A collection of all timestamp related values contained within Dune Response""" 88 | 89 | submitted_at: datetime 90 | execution_started_at: Optional[datetime] 91 | execution_ended_at: Optional[datetime] 92 | # Expires only exists when we have result data 93 | expires_at: Optional[datetime] 94 | # only exists for cancelled executions 95 | cancelled_at: Optional[datetime] 96 | 97 | @classmethod 98 | def from_dict(cls, data: dict[str, Any]) -> TimeData: 99 | """Constructor from dictionary. See unit test for sample input.""" 100 | start = data.get("execution_started_at") 101 | end = data.get("execution_ended_at") 102 | expires = data.get("expires_at") 103 | cancelled = data.get("cancelled_at") 104 | return cls( 105 | submitted_at=parse(data["submitted_at"]), 106 | expires_at=None if expires is None else parse(expires), 107 | execution_started_at=None if start is None else parse(start), 108 | execution_ended_at=None if end is None else parse(end), 109 | cancelled_at=None if cancelled is None else parse(cancelled), 110 | ) 111 | 112 | 113 | @dataclass 114 | class ExecutionError: 115 | """ 116 | Representation of Execution Error Response: 117 | 118 | Example: 119 | { 120 | "type":"FAILED_TYPE_EXECUTION_FAILED", 121 | "message":"line 24:13: Binary literal can only contain hexadecimal digits", 122 | "metadata":{"line":24,"column":13} 123 | } 124 | """ 125 | 126 | type: str 127 | message: str 128 | metadata: str 129 | 130 | @classmethod 131 | def from_dict(cls, data: dict[str, str]) -> ExecutionError: 132 | """Constructs an instance from a dict""" 133 | return cls( 134 | type=data.get("type", "unknown"), 135 | message=data.get("message", "unknown"), 136 | metadata=data.get("metadata", "unknown"), 137 | ) 138 | 139 | 140 | @dataclass 141 | class ExecutionStatusResponse: 142 | """ 143 | Representation of Response from Dune's [Get] Execution Status endpoint 144 | """ 145 | 146 | execution_id: str 147 | query_id: int 148 | state: ExecutionState 149 | times: TimeData 150 | queue_position: Optional[int] 151 | # this will be present when the query execution completes 152 | result_metadata: Optional[ResultMetadata] 153 | error: Optional[ExecutionError] 154 | 155 | @classmethod 156 | def from_dict(cls, data: dict[str, Any]) -> ExecutionStatusResponse: 157 | """Constructor from dictionary. See unit test for sample input.""" 158 | dct: Optional[MetaData] = data.get("result_metadata") 159 | error: Optional[dict[str, str]] = data.get("error") 160 | return cls( 161 | execution_id=data["execution_id"], 162 | query_id=int(data["query_id"]), 163 | queue_position=data.get("queue_position"), 164 | state=ExecutionState(data["state"]), 165 | result_metadata=ResultMetadata.from_dict(dct) if dct else None, 166 | times=TimeData.from_dict(data), # Sending the entire data dict 167 | error=ExecutionError.from_dict(error) if error else None, 168 | ) 169 | 170 | def __str__(self) -> str: 171 | if self.state == ExecutionState.PENDING: 172 | return f"{self.state} (queue position: {self.queue_position})" 173 | if self.state == ExecutionState.FAILED: 174 | return ( 175 | f"{self.state}: execution_id={self.execution_id}, " 176 | f"query_id={self.query_id}, times={self.times}" 177 | ) 178 | 179 | return f"{self.state}" 180 | 181 | 182 | @dataclass 183 | class ResultMetadata: 184 | """ 185 | Representation of Dune's Result Metadata from [Get] Query Results endpoint 186 | """ 187 | 188 | # pylint: disable=too-many-instance-attributes 189 | 190 | column_names: list[str] 191 | column_types: list[str] 192 | row_count: int 193 | result_set_bytes: int 194 | total_row_count: int 195 | total_result_set_bytes: int 196 | datapoint_count: int 197 | pending_time_millis: Optional[int] 198 | execution_time_millis: int 199 | 200 | @classmethod 201 | def from_dict(cls, data: dict[str, Any]) -> ResultMetadata: 202 | """Constructor from dictionary. See unit test for sample input.""" 203 | assert isinstance(data["column_names"], list) 204 | pending_time = data.get("pending_time_millis", None) 205 | return cls( 206 | column_names=data["column_names"], 207 | column_types=data["column_types"], 208 | row_count=int(data["total_row_count"]), 209 | result_set_bytes=int(data["result_set_bytes"]), 210 | total_row_count=int(data["total_row_count"]), 211 | total_result_set_bytes=int(data["result_set_bytes"]), 212 | datapoint_count=int(data["datapoint_count"]), 213 | pending_time_millis=int(pending_time) if pending_time else None, 214 | execution_time_millis=int(data["execution_time_millis"]), 215 | ) 216 | 217 | def __add__(self, other: ResultMetadata) -> ResultMetadata: 218 | """ 219 | Enables combining results by updating the metadata associated to 220 | an execution by using the `+` operator. 221 | """ 222 | assert other is not None 223 | 224 | self.row_count += other.row_count 225 | self.result_set_bytes += other.result_set_bytes 226 | self.datapoint_count += other.datapoint_count 227 | return self 228 | 229 | 230 | RowData = List[Dict[str, Any]] 231 | MetaData = Dict[str, Union[int, List[str]]] 232 | 233 | 234 | @dataclass 235 | class ExecutionResultCSV: 236 | """ 237 | Representation of a raw `result` in CSV format 238 | this payload can be passed directly to 239 | csv.reader(data) or 240 | pandas.read_csv(data) 241 | """ 242 | 243 | data: BytesIO # includes all CSV rows, including the header row. 244 | next_uri: Optional[str] = None 245 | next_offset: Optional[int] = None 246 | 247 | def __add__(self, other: ExecutionResultCSV) -> ExecutionResultCSV: 248 | assert other is not None 249 | assert other.data is not None 250 | 251 | self.next_uri = other.next_uri 252 | self.next_offset = other.next_offset 253 | 254 | # Get to the end of the current CSV 255 | self.data.seek(0, SEEK_END) 256 | 257 | # Skip the first line of the new CSV, which contains the header 258 | other.data.readline() 259 | 260 | # Append the rest of the content from `other` into current one 261 | self.data.write(other.data.read()) 262 | 263 | # Move the cursor back to the start of the CSV 264 | self.data.seek(0) 265 | 266 | return self 267 | 268 | 269 | @dataclass 270 | class ExecutionResult: 271 | """Representation of `result` field of a Dune ResultsResponse""" 272 | 273 | rows: list[DuneRecord] 274 | metadata: ResultMetadata 275 | 276 | @classmethod 277 | def from_dict(cls, data: dict[str, RowData | MetaData]) -> ExecutionResult: 278 | """Constructor from dictionary. See unit test for sample input.""" 279 | assert isinstance(data["rows"], list) 280 | assert isinstance(data["metadata"], dict) 281 | return cls( 282 | rows=data["rows"], 283 | metadata=ResultMetadata.from_dict(data["metadata"]), 284 | ) 285 | 286 | def __add__(self, other: ExecutionResult) -> ExecutionResult: 287 | """ 288 | Enables combining results using the `+` operator. 289 | """ 290 | self.rows.extend(other.rows) 291 | self.metadata += other.metadata 292 | 293 | return self 294 | 295 | 296 | ResultData = Dict[str, Union[RowData, MetaData]] 297 | 298 | 299 | @dataclass 300 | class ResultsResponse: 301 | """ 302 | Representation of Response from Dune's [Get] Query Results endpoint 303 | """ 304 | 305 | execution_id: str 306 | query_id: int 307 | state: ExecutionState 308 | times: TimeData 309 | # optional because it will only be present when the query execution completes 310 | result: Optional[ExecutionResult] 311 | next_uri: Optional[str] 312 | next_offset: Optional[int] 313 | 314 | @classmethod 315 | def from_dict(cls, data: dict[str, str | int | ResultData]) -> ResultsResponse: 316 | """Constructor from dictionary. See unit test for sample input.""" 317 | assert isinstance(data["execution_id"], str) 318 | assert isinstance(data["query_id"], int) 319 | assert isinstance(data["state"], str) 320 | result = data.get("result", {}) 321 | assert isinstance(result, dict) 322 | next_uri = data.get("next_uri") 323 | assert isinstance(next_uri, str) or next_uri is None 324 | next_offset = data.get("next_offset") 325 | assert isinstance(next_offset, int) or next_offset is None 326 | return cls( 327 | execution_id=data["execution_id"], 328 | query_id=int(data["query_id"]), 329 | state=ExecutionState(data["state"]), 330 | times=TimeData.from_dict(data), 331 | result=ExecutionResult.from_dict(result) if result else None, 332 | next_uri=next_uri, 333 | next_offset=next_offset, 334 | ) 335 | 336 | def get_rows(self) -> list[DuneRecord]: 337 | """ 338 | Absorbs the Optional check and returns the result rows. 339 | When execution is a non-complete terminal state, returns empty list. 340 | """ 341 | 342 | if self.state == ExecutionState.COMPLETED: 343 | assert self.result is not None, f"No Results on completed execution {self}" 344 | return self.result.rows 345 | 346 | log.info(f"execution {self.state} returning empty list") 347 | return [] 348 | 349 | def __add__(self, other: ResultsResponse) -> ResultsResponse: 350 | """ 351 | Enables combining results using the `+` operator. 352 | """ 353 | assert self.execution_id == other.execution_id 354 | assert self.result is not None 355 | assert other.result is not None 356 | self.result += other.result 357 | self.next_uri = other.next_uri 358 | self.next_offset = other.next_offset 359 | return self 360 | 361 | 362 | @dataclass 363 | class CreateTableResult(DataClassJsonMixin): 364 | """ 365 | Data type returned by table/create operation 366 | """ 367 | 368 | namespace: str 369 | table_name: str 370 | full_name: str 371 | example_query: str 372 | already_existed: bool 373 | message: str 374 | 375 | 376 | @dataclass 377 | class InsertTableResult(DataClassJsonMixin): 378 | """ 379 | Data type returned by table/insert operation 380 | """ 381 | 382 | rows_written: int 383 | bytes_written: int 384 | 385 | 386 | @dataclass 387 | class DeleteTableResult(DataClassJsonMixin): 388 | """ 389 | Data type returned by table/delete operation 390 | """ 391 | 392 | message: str 393 | 394 | 395 | @dataclass 396 | class ClearTableResult(DataClassJsonMixin): 397 | """ 398 | Data type returned by table/clear operation 399 | """ 400 | 401 | message: str 402 | -------------------------------------------------------------------------------- /dune_client/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/duneanalytics/dune-client/1d466e80cd9fe2280d6638e43b94b49067a3817f/dune_client/py.typed -------------------------------------------------------------------------------- /dune_client/query.py: -------------------------------------------------------------------------------- 1 | """ 2 | Data Classes Representing a Dune Query 3 | """ 4 | 5 | from __future__ import annotations 6 | import urllib.parse 7 | from dataclasses import dataclass 8 | from typing import Optional, List, Dict, Union, Any 9 | 10 | from dune_client.types import QueryParameter 11 | 12 | 13 | def parse_query_object_or_id( 14 | query: Union[QueryBase, str, int], 15 | ) -> tuple[dict[str, Any] | None, int]: 16 | """ 17 | Users are allowed to pass QueryBase or ID into some functions. 18 | This method handles both scenarios, returning a pair of the form (params, query_id) 19 | """ 20 | if isinstance(query, QueryBase): 21 | params = {f"params.{p.key}": p.to_dict()["value"] for p in query.parameters()} 22 | query_id = query.query_id 23 | else: 24 | params = None 25 | query_id = int(query) 26 | return params, query_id 27 | 28 | 29 | @dataclass 30 | class QueryBase: 31 | """Basic data structure constituting a Dune Analytics Query.""" 32 | 33 | query_id: int 34 | name: str = "unnamed" 35 | params: Optional[List[QueryParameter]] = None 36 | 37 | def base_url(self) -> str: 38 | """Returns a link to query results excluding fixed parameters""" 39 | return f"https://dune.com/queries/{self.query_id}" 40 | 41 | def parameters(self) -> List[QueryParameter]: 42 | """Non-null version of self.params""" 43 | return self.params or [] 44 | 45 | def url(self) -> str: 46 | """Returns a parameterized link to the query""" 47 | # Include variable parameters in the URL so they are set 48 | params = "&".join([f"{p.key}={p.value}" for p in self.parameters()]) 49 | if params: 50 | return "?".join( 51 | [self.base_url(), urllib.parse.quote_plus(params, safe="=&?")] 52 | ) 53 | return self.base_url() 54 | 55 | def __hash__(self) -> int: 56 | """ 57 | This contains the query ID and the values of relevant parameters. 58 | Thus, it is unique for caching purposes 59 | """ 60 | return self.url().__hash__() 61 | 62 | def request_format(self) -> Dict[str, Union[Dict[str, str], str, None]]: 63 | """Transforms Query objects to params to pass in API""" 64 | return { 65 | "query_parameters": {p.key: p.to_dict()["value"] for p in self.parameters()} 66 | } 67 | 68 | 69 | @dataclass 70 | class QueryMeta: # pylint: disable=too-many-instance-attributes 71 | """ 72 | Data class containing meta content about the query 73 | """ 74 | 75 | description: str 76 | tags: list[str] 77 | version: int 78 | engine: str 79 | is_private: bool 80 | is_archived: bool 81 | is_unsaved: bool 82 | owner: str 83 | 84 | 85 | @dataclass 86 | class DuneQuery: 87 | """ 88 | Enriched class representing all data constituting a DuneQuery 89 | Modeling the CRUD operation response for `get_query` 90 | """ 91 | 92 | base: QueryBase 93 | meta: QueryMeta 94 | sql: str 95 | 96 | @classmethod 97 | def from_dict(cls, data: dict[str, Any]) -> DuneQuery: 98 | """Constructor from json object""" 99 | return cls( 100 | base=QueryBase( 101 | query_id=int(data["query_id"]), 102 | name=data["name"], 103 | params=[ 104 | QueryParameter.from_dict(param) for param in data["parameters"] 105 | ], 106 | ), 107 | meta=QueryMeta( 108 | description=data["description"], 109 | tags=data["tags"], 110 | version=data["version"], 111 | engine=data["query_engine"], 112 | is_private=data["is_private"], 113 | is_archived=data["is_archived"], 114 | is_unsaved=data["is_unsaved"], 115 | owner=data["owner"], 116 | ), 117 | sql=data["query_sql"], 118 | ) 119 | -------------------------------------------------------------------------------- /dune_client/types.py: -------------------------------------------------------------------------------- 1 | """ 2 | These types were primarily copied from: 3 | https://github.com/bh2smith/duneapi/blob/v4.0.0/duneapi/types.py 4 | with small adjustments (removing Options from QueryParameter) 5 | """ 6 | 7 | from __future__ import annotations 8 | 9 | import re 10 | from datetime import datetime 11 | from enum import Enum 12 | from typing import Any, Dict 13 | 14 | from dune_client.util import postgres_date 15 | 16 | DuneRecord = Dict[str, Any] 17 | 18 | 19 | # pylint: disable=too-few-public-methods 20 | class Address: 21 | """ 22 | Class representing Ethereum Address as a hexadecimal string of length 42. 23 | The string must begin with '0x' and the other 40 characters 24 | are digits 0-9 or letters a-f. Upon creation (from string) addresses 25 | are validated and stored in their check-summed format. 26 | """ 27 | 28 | def __init__(self, address: str): 29 | # Dune uses \x instead of 0x (i.e. bytea instead of hex string) 30 | # This is just a courtesy to query writers, 31 | # so they don't have to convert all addresses to hex strings manually 32 | address = address.replace("\\x", "0x") 33 | if Address._is_valid(address): 34 | self.address: str = address.lower() 35 | else: 36 | raise ValueError(f"Invalid Ethereum Address {address}") 37 | 38 | def __str__(self) -> str: 39 | return str(self.address) 40 | 41 | def __eq__(self, other: object) -> bool: 42 | if isinstance(other, Address): 43 | return self.address == other.address 44 | return False 45 | 46 | def __lt__(self, other: object) -> bool: 47 | if isinstance(other, Address): 48 | return str(self).lower() < str(other).lower() 49 | return False 50 | 51 | def __hash__(self) -> int: 52 | return self.address.__hash__() 53 | 54 | @classmethod 55 | def zero(cls) -> Address: 56 | """Returns Null Ethereum Address""" 57 | return cls("0x0000000000000000000000000000000000000000") 58 | 59 | @classmethod 60 | def from_int(cls, num: int) -> Address: 61 | """ 62 | Construct an address from int. 63 | Used for testing, so that 123 -> "0x0000000000000000000000000000000000000123" 64 | """ 65 | return cls(f"0x{str(num).rjust(40, '0')}") 66 | 67 | @staticmethod 68 | def _is_valid(address: str) -> bool: 69 | match_result = re.match( 70 | pattern=r"^(0x)?[0-9a-f]{40}$", string=address, flags=re.IGNORECASE 71 | ) 72 | return match_result is not None 73 | 74 | 75 | class ParameterType(Enum): 76 | """ 77 | Enum of the 4 distinct dune parameter types 78 | """ 79 | 80 | TEXT = "text" 81 | NUMBER = "number" 82 | DATE = "datetime" 83 | ENUM = "enum" 84 | 85 | @classmethod 86 | def from_string(cls, type_str: str) -> ParameterType: 87 | """ 88 | Attempts to parse Parameter from string. 89 | returns None is no match 90 | """ 91 | patterns = { 92 | r"text": cls.TEXT, 93 | r"number": cls.NUMBER, 94 | r"date": cls.DATE, 95 | r"enum": cls.ENUM, 96 | r"list": cls.ENUM, 97 | } 98 | for pattern, param in patterns.items(): 99 | if re.match(pattern, type_str, re.IGNORECASE): 100 | return param 101 | raise ValueError(f"could not parse Network from '{type_str}'") 102 | 103 | 104 | class QueryParameter: 105 | """Class whose instances are Dune Compatible Query Parameters""" 106 | 107 | def __init__( 108 | self, 109 | name: str, 110 | parameter_type: ParameterType, 111 | value: Any, 112 | ): 113 | self.key: str = name 114 | self.type: ParameterType = parameter_type 115 | self.value = value 116 | 117 | def __eq__(self, other: object) -> bool: 118 | if not isinstance(other, QueryParameter): 119 | return NotImplemented 120 | return all( 121 | [ 122 | self.key == other.key, 123 | self.value == other.value, 124 | self.type.value == other.type.value, 125 | ] 126 | ) 127 | 128 | @classmethod 129 | def text_type(cls, name: str, value: str) -> QueryParameter: 130 | """Constructs a Query parameter of type text""" 131 | return cls(name, ParameterType.TEXT, value) 132 | 133 | @classmethod 134 | def number_type(cls, name: str, value: int | float) -> QueryParameter: 135 | """Constructs a Query parameter of type number""" 136 | return cls(name, ParameterType.NUMBER, value) 137 | 138 | @classmethod 139 | def date_type(cls, name: str, value: datetime | str) -> QueryParameter: 140 | """ 141 | Constructs a Query parameter of type date. 142 | For convenience, we allow proper datetime type, or string 143 | """ 144 | if isinstance(value, str): 145 | value = postgres_date(value) 146 | return cls(name, ParameterType.DATE, value) 147 | 148 | @classmethod 149 | def enum_type(cls, name: str, value: str) -> QueryParameter: 150 | """Constructs a Query parameter of type number""" 151 | return cls(name, ParameterType.ENUM, value) 152 | 153 | def value_str(self) -> str: 154 | """Returns string value of parameter""" 155 | if self.type in (ParameterType.TEXT, ParameterType.NUMBER, ParameterType.ENUM): 156 | return str(self.value) 157 | if self.type == ParameterType.DATE: 158 | # This is the postgres string format of timestamptz 159 | return str(self.value.strftime("%Y-%m-%d %H:%M:%S")) 160 | raise TypeError(f"Type {self.type} not recognized!") 161 | 162 | def to_dict(self) -> dict[str, str]: 163 | """Converts QueryParameter into string json format accepted by Dune API""" 164 | results: dict[str, str] = { 165 | "key": self.key, 166 | "type": self.type.value, 167 | "value": self.value_str(), 168 | } 169 | return results 170 | 171 | @classmethod 172 | def from_dict(cls, obj: dict[str, Any]) -> QueryParameter: 173 | """ 174 | Constructs Query Parameters from json. 175 | TODO - this could probably be done similar to the __init__ method of MetaData 176 | """ 177 | name, value = obj["key"], obj["value"] 178 | p_type = ParameterType.from_string(obj["type"]) 179 | if p_type == ParameterType.DATE: 180 | return cls.date_type(name, value) 181 | if p_type == ParameterType.TEXT: 182 | assert isinstance(value, str) 183 | return cls.text_type(name, value) 184 | if p_type == ParameterType.NUMBER: 185 | if isinstance(value, str): 186 | value = float(value) if "." in value else int(value) 187 | return cls.number_type(name, value) 188 | if p_type == ParameterType.ENUM: 189 | return cls.enum_type(name, value) 190 | raise ValueError(f"Could not parse Query parameter from {obj}") 191 | 192 | def __str__(self) -> str: 193 | # For less cryptic logging. 194 | return ( 195 | f"Parameter(" 196 | f"name={self.key}, " 197 | f"value={self.value}, " 198 | f"type={self.type.value})" 199 | ) 200 | 201 | def __repr__(self) -> str: 202 | return str(self) 203 | -------------------------------------------------------------------------------- /dune_client/util.py: -------------------------------------------------------------------------------- 1 | """Utility methods for package.""" 2 | 3 | from datetime import datetime, timezone 4 | import importlib 5 | from typing import Optional 6 | 7 | DUNE_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" 8 | 9 | 10 | def postgres_date(date_str: str) -> datetime: 11 | """Parse a postgres compatible date string into datetime object""" 12 | return datetime.strptime(date_str, DUNE_DATE_FORMAT) 13 | 14 | 15 | def get_package_version(package_name: str) -> Optional[str]: 16 | """ 17 | Returns the package version by `package_name` using the importlib.metadata module 18 | which is available in Python 3.8 and later. 19 | """ 20 | try: 21 | return importlib.metadata.version(package_name) 22 | except importlib.metadata.PackageNotFoundError: 23 | return None 24 | 25 | 26 | def age_in_hours(timestamp: datetime) -> float: 27 | """ 28 | Returns the time (in hours) between now and `timestamp` 29 | """ 30 | result_age = datetime.now(timezone.utc) - timestamp 31 | return result_age.total_seconds() / (60 * 60) 32 | -------------------------------------------------------------------------------- /dune_client/viz/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/duneanalytics/dune-client/1d466e80cd9fe2280d6638e43b94b49067a3817f/dune_client/viz/__init__.py -------------------------------------------------------------------------------- /dune_client/viz/graphs.py: -------------------------------------------------------------------------------- 1 | """ 2 | Functions you can call to make different graphs 3 | """ 4 | 5 | from typing import Dict, Union 6 | 7 | # https://github.com/plotly/colorlover/issues/35 8 | import colorlover as cl # type: ignore[import-untyped] 9 | import pandas as pd 10 | import plotly.graph_objects as go # type: ignore[import-untyped] 11 | from plotly.graph_objs import Figure # type: ignore[import-untyped] 12 | 13 | 14 | # function to create Sankey diagram 15 | def create_sankey( 16 | query_result: pd.DataFrame, 17 | predefined_colors: Dict[str, str], 18 | columns: Dict[str, str], 19 | viz_config: Dict[str, Union[int, float]], 20 | title: str = "unnamed", 21 | ) -> Figure: 22 | """ 23 | Creates a Sankey diagram based on input query_result, 24 | which must contain source, target, value columns. 25 | Column names don't have to be exact same but there must be 26 | these three columns conceptually and value column must be numeric. 27 | """ 28 | # Check if the dataframe contains required columns 29 | required_columns = [columns["source"], columns["target"], columns["value"]] 30 | for col in required_columns: 31 | if col not in query_result.columns: 32 | raise ValueError(f"Error: The dataframe is missing the '{col}' column") 33 | 34 | # Check if 'value' column is numeric 35 | if not pd.api.types.is_numeric_dtype(query_result[columns["value"]]): 36 | raise ValueError("Error: The 'value' column must be numeric") 37 | 38 | # preprocess query result dataframe 39 | all_nodes = list( 40 | pd.concat( 41 | [query_result[columns["source"]], query_result[columns["target"]]] 42 | ).unique() 43 | ) 44 | # In Sankey, 'source' and 'target' must be indices. Thus, you need to map projects to indices. 45 | query_result["source_idx"] = query_result[columns["source"]].map(all_nodes.index) 46 | query_result["target_idx"] = query_result[columns["target"]].map(all_nodes.index) 47 | 48 | # create color map for Sankey 49 | colors = cl.scales["12"]["qual"]["Set3"] # default color 50 | color_map = {} 51 | for node in all_nodes: 52 | for name, color in predefined_colors.items(): 53 | if name.lower() in node.lower(): # check if name exists in the node name 54 | color_map[node] = color 55 | break 56 | else: 57 | color_map[node] = colors[ 58 | len(color_map) % len(colors) 59 | ] # default color assignment 60 | 61 | fig = go.Figure( 62 | go.Sankey( 63 | node={ 64 | "pad": viz_config["node_pad"], 65 | "thickness": viz_config["node_thickness"], 66 | "line": {"color": "black", "width": viz_config["node_line_width"]}, 67 | "label": all_nodes, 68 | "color": [ 69 | color_map.get(node, "blue") for node in all_nodes 70 | ], # customize node color 71 | }, 72 | link={ 73 | "source": query_result["source_idx"], 74 | "target": query_result["target_idx"], 75 | "value": query_result[columns["value"]], 76 | "color": [ 77 | color_map.get(query_result[columns["source"]].iloc[i], "black") 78 | for i in range(len(query_result)) 79 | ], # customize link color 80 | }, 81 | ) 82 | ) 83 | fig.update_layout( 84 | title_text=title, 85 | font_size=viz_config["font_size"], 86 | height=viz_config["figure_height"], 87 | width=viz_config["figure_width"], 88 | ) 89 | 90 | return fig 91 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools >= 48", 4 | "setuptools_scm[toml] >= 6.2", 5 | "setuptools_scm_git_archive", 6 | "wheel >= 0.29.0", 7 | ] 8 | build-backend = 'setuptools.build_meta' 9 | 10 | [tool.setuptools_scm] 11 | write_to = "_version.py" 12 | git_describe_command = "git describe --dirty --tags --long --match v* --first-parent" -------------------------------------------------------------------------------- /requirements/dev.txt: -------------------------------------------------------------------------------- 1 | -r prod.txt 2 | black>=23.7.0 3 | pandas>=1.0.0 4 | pandas-stubs>=1.0.0 5 | pylint>=2.17.5 6 | pytest>=7.4.1 7 | python-dotenv>=1.0.0 8 | mypy>=1.5.1 9 | aiounittest>=1.4.2 10 | colorlover>=0.3.0 11 | plotly>=5.9.0 12 | -------------------------------------------------------------------------------- /requirements/prod.txt: -------------------------------------------------------------------------------- 1 | aiohttp>=3.8.5 2 | dataclasses-json==0.6.4 3 | types-python-dateutil>=2.8.19.14 4 | types-PyYAML>=6.0.12.11 5 | types-requests>=2.28.0 6 | types-setuptools>=68.2.0.0 7 | python-dateutil>=2.8.2 8 | requests>=2.28.0 9 | ndjson>=0.3.1 10 | Deprecated>=1.2.14 11 | types-Deprecated==1.2.9.3 12 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = dune_client 3 | description = A simple framework for interacting with Dune Analytics official API service. 4 | long_description = file: README.md 5 | long_description_content_type = text/markdown; charset=UTF-8 6 | url = https://github.com/duneanalytics/dune-client 7 | author = Benjamin H. Smith & Dune Analytics 8 | author_email = ben@cow.fi 9 | license = Apache License Version 2.0 10 | license_files = LICENSE 11 | classifiers = 12 | Programming Language :: Python :: 3 13 | License :: OSI Approved :: Apache Software License 14 | Operating System :: OS Independent 15 | project_urls = 16 | Tracker = https://github.com/duneanalytics/dune-client/issues 17 | 18 | [options] 19 | zip_safe = False 20 | packages=find: 21 | platforms = any 22 | install_requires = 23 | aiohttp>=3.8.3 24 | dataclasses-json>=0.6.4 25 | types-python-dateutil>=2.8.19 26 | types-PyYAML>=6.0.11 27 | types-requests>=2.28.0 28 | types-Deprecated>=1.2.9.3 29 | types-setuptools>=68.2.0.0 30 | python-dateutil>=2.8.2 31 | requests>=2.28.0 32 | ndjson>=0.3.1 33 | Deprecated>=1.2.0 34 | python_requires = >=3.8 35 | setup_requires = 36 | setuptools_scm 37 | 38 | [options.packages.find] 39 | exclude = 40 | tests 41 | tests.* 42 | example 43 | example.* 44 | 45 | [options.package_data] 46 | dune_client = py.typed 47 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | if __name__ == "__main__": 4 | setuptools.setup() 5 | -------------------------------------------------------------------------------- /tests/e2e/test_async_client.py: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | 4 | import aiounittest 5 | import dotenv 6 | import pandas 7 | 8 | from dune_client.client_async import AsyncDuneClient 9 | from dune_client.query import QueryBase 10 | 11 | 12 | class TestDuneClient(aiounittest.AsyncTestCase): 13 | def setUp(self) -> None: 14 | self.query = QueryBase(name="Sample Query", query_id=1215383) 15 | self.multi_rows_query = QueryBase( 16 | name="Query that returns multiple rows", 17 | query_id=3435763, 18 | ) 19 | dotenv.load_dotenv() 20 | self.valid_api_key = os.environ["DUNE_API_KEY"] 21 | 22 | async def test_disconnect(self): 23 | dune = AsyncDuneClient(self.valid_api_key) 24 | await dune.connect() 25 | results = (await dune.refresh(self.query)).get_rows() 26 | self.assertGreater(len(results), 0) 27 | await dune.disconnect() 28 | self.assertTrue(dune._session.closed) 29 | 30 | async def test_refresh_context_manager_singleton(self): 31 | dune = AsyncDuneClient(self.valid_api_key) 32 | async with dune as cl: 33 | results = (await cl.refresh(self.query)).get_rows() 34 | self.assertGreater(len(results), 0) 35 | 36 | async def test_refresh_context_manager(self): 37 | async with AsyncDuneClient(self.valid_api_key) as cl: 38 | results = (await cl.refresh(self.query)).get_rows() 39 | self.assertGreater(len(results), 0) 40 | 41 | async def test_refresh_with_pagination(self): 42 | # Arrange 43 | async with AsyncDuneClient(self.valid_api_key) as cl: 44 | # Act 45 | results = (await cl.refresh(self.multi_rows_query, batch_size=1)).get_rows() 46 | 47 | # Assert 48 | self.assertEqual( 49 | results, 50 | [ 51 | {"number": 1}, 52 | {"number": 2}, 53 | {"number": 3}, 54 | {"number": 4}, 55 | {"number": 5}, 56 | ], 57 | ) 58 | 59 | async def test_refresh_with_filters(self): 60 | # Arrange 61 | async with AsyncDuneClient(self.valid_api_key) as cl: 62 | # Act 63 | results = ( 64 | await cl.refresh(self.multi_rows_query, filters="number < 3") 65 | ).get_rows() 66 | 67 | # Assert 68 | self.assertEqual( 69 | results, 70 | [ 71 | {"number": 1}, 72 | {"number": 2}, 73 | ], 74 | ) 75 | 76 | async def test_refresh_csv_with_pagination(self): 77 | # Arrange 78 | async with AsyncDuneClient(self.valid_api_key) as cl: 79 | # Act 80 | result_csv = await cl.refresh_csv(self.multi_rows_query, batch_size=1) 81 | 82 | # Assert 83 | self.assertEqual( 84 | pandas.read_csv(result_csv.data).to_dict(orient="records"), 85 | [ 86 | {"number": 1}, 87 | {"number": 2}, 88 | {"number": 3}, 89 | {"number": 4}, 90 | {"number": 5}, 91 | ], 92 | ) 93 | 94 | async def test_refresh_csv_with_filters(self): 95 | # Arrange 96 | async with AsyncDuneClient(self.valid_api_key) as cl: 97 | # Act 98 | result_csv = await cl.refresh_csv( 99 | self.multi_rows_query, filters="number < 3" 100 | ) 101 | 102 | # Assert 103 | self.assertEqual( 104 | pandas.read_csv(result_csv.data).to_dict(orient="records"), 105 | [ 106 | {"number": 1}, 107 | {"number": 2}, 108 | ], 109 | ) 110 | 111 | @unittest.skip("Large performance tier doesn't currently work.") 112 | async def test_refresh_context_manager_performance_large(self): 113 | async with AsyncDuneClient(self.valid_api_key) as cl: 114 | results = (await cl.refresh(self.query, performance="large")).get_rows() 115 | self.assertGreater(len(results), 0) 116 | 117 | async def test_get_latest_result_with_query_object(self): 118 | async with AsyncDuneClient(self.valid_api_key) as cl: 119 | results = (await cl.get_latest_result(self.query)).get_rows() 120 | self.assertGreater(len(results), 0) 121 | 122 | async def test_get_latest_result_with_query_id(self): 123 | async with AsyncDuneClient(self.valid_api_key) as cl: 124 | results = (await cl.get_latest_result(self.query.query_id)).get_rows() 125 | self.assertGreater(len(results), 0) 126 | 127 | 128 | if __name__ == "__main__": 129 | unittest.main() 130 | -------------------------------------------------------------------------------- /tests/e2e/test_client.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import os 3 | import time 4 | import unittest 5 | 6 | import dotenv 7 | import pandas 8 | 9 | from dune_client.models import ( 10 | ExecutionState, 11 | ExecutionResponse, 12 | ExecutionStatusResponse, 13 | DuneError, 14 | InsertTableResult, 15 | CreateTableResult, 16 | DeleteTableResult, 17 | ClearTableResult, 18 | ) 19 | from dune_client.types import QueryParameter 20 | from dune_client.client import DuneClient 21 | from dune_client.query import QueryBase 22 | 23 | dotenv.load_dotenv() 24 | 25 | 26 | class TestDuneClient(unittest.TestCase): 27 | def setUp(self) -> None: 28 | self.query = QueryBase( 29 | name="Sample Query", 30 | query_id=1215383, 31 | params=[ 32 | # These are the queries default parameters. 33 | QueryParameter.text_type(name="TextField", value="Plain Text"), 34 | QueryParameter.number_type(name="NumberField", value=3.1415926535), 35 | QueryParameter.date_type(name="DateField", value="2022-05-04 00:00:00"), 36 | QueryParameter.enum_type(name="ListField", value="Option 1"), 37 | ], 38 | ) 39 | self.multi_rows_query = QueryBase( 40 | name="Query that returns multiple rows", 41 | query_id=3435763, 42 | ) 43 | self.valid_api_key = os.environ["DUNE_API_KEY"] 44 | 45 | def copy_query_and_change_parameters(self) -> QueryBase: 46 | new_query = copy.copy(self.query) 47 | new_query.params = [ 48 | # Using all different values for parameters. 49 | QueryParameter.text_type(name="TextField", value="different word"), 50 | QueryParameter.number_type(name="NumberField", value=22), 51 | QueryParameter.date_type(name="DateField", value="1991-01-01 00:00:00"), 52 | QueryParameter.enum_type(name="ListField", value="Option 2"), 53 | ] 54 | self.assertNotEqual(self.query.parameters(), new_query.parameters()) 55 | return new_query 56 | 57 | def test_from_env_constructor(self): 58 | try: 59 | DuneClient.from_env() 60 | except KeyError: 61 | self.fail("DuneClient.from_env raised unexpectedly!") 62 | 63 | def test_get_execution_status(self): 64 | query = QueryBase(name="No Name", query_id=1276442, params=[]) 65 | dune = DuneClient(self.valid_api_key) 66 | job_id = dune.execute_query(query).execution_id 67 | status = dune.get_execution_status(job_id) 68 | self.assertTrue( 69 | status.state in [ExecutionState.EXECUTING, ExecutionState.PENDING] 70 | ) 71 | 72 | def test_run_query(self): 73 | dune = DuneClient(self.valid_api_key) 74 | results = dune.run_query(self.query).get_rows() 75 | self.assertGreater(len(results), 0) 76 | 77 | def test_run_query_paginated(self): 78 | # Arrange 79 | dune = DuneClient(self.valid_api_key) 80 | 81 | # Act 82 | results = dune.run_query(self.multi_rows_query, batch_size=1).get_rows() 83 | 84 | # Assert 85 | self.assertEqual( 86 | results, 87 | [ 88 | {"number": 1}, 89 | {"number": 2}, 90 | {"number": 3}, 91 | {"number": 4}, 92 | {"number": 5}, 93 | ], 94 | ) 95 | 96 | def test_run_query_with_filters(self): 97 | # Arrange 98 | dune = DuneClient(self.valid_api_key) 99 | 100 | # Act 101 | results = dune.run_query(self.multi_rows_query, filters="number < 3").get_rows() 102 | 103 | # Assert 104 | self.assertEqual( 105 | results, 106 | [ 107 | {"number": 1}, 108 | {"number": 2}, 109 | ], 110 | ) 111 | 112 | def test_run_query_performance_large(self): 113 | dune = DuneClient(self.valid_api_key) 114 | results = dune.run_query(self.query, performance="large").get_rows() 115 | self.assertGreater(len(results), 0) 116 | 117 | def test_run_query_dataframe(self): 118 | dune = DuneClient(self.valid_api_key) 119 | pd = dune.run_query_dataframe(self.query) 120 | self.assertGreater(len(pd), 0) 121 | 122 | def test_parameters_recognized(self): 123 | new_query = self.copy_query_and_change_parameters() 124 | dune = DuneClient(self.valid_api_key) 125 | results = dune.run_query(new_query) 126 | self.assertEqual( 127 | results.get_rows(), 128 | [ 129 | { 130 | "text_field": "different word", 131 | "number_field": 22, 132 | "date_field": "1991-01-01 00:00:00", 133 | "list_field": "Option 2", 134 | } 135 | ], 136 | ) 137 | 138 | def test_endpoints(self): 139 | dune = DuneClient(self.valid_api_key) 140 | execution_response = dune.execute_query(self.query) 141 | self.assertIsInstance(execution_response, ExecutionResponse) 142 | job_id = execution_response.execution_id 143 | status = dune.get_execution_status(job_id) 144 | self.assertIsInstance(status, ExecutionStatusResponse) 145 | while dune.get_execution_status(job_id).state != ExecutionState.COMPLETED: 146 | time.sleep(1) 147 | results = dune.get_execution_results(job_id).result.rows 148 | self.assertGreater(len(results), 0) 149 | 150 | def test_cancel_execution(self): 151 | dune = DuneClient(self.valid_api_key) 152 | query = QueryBase( 153 | name="Long Running Query", 154 | query_id=1229120, 155 | ) 156 | execution_response = dune.execute_query(query) 157 | job_id = execution_response.execution_id 158 | # POST Cancellation 159 | success = dune.cancel_execution(job_id) 160 | self.assertTrue(success) 161 | 162 | results = dune.get_execution_results(job_id) 163 | self.assertEqual(results.state, ExecutionState.CANCELLED) 164 | 165 | def test_invalid_api_key_error(self): 166 | dune = DuneClient(api_key="Invalid Key") 167 | with self.assertRaises(DuneError) as err: 168 | dune.execute_query(self.query) 169 | self.assertEqual( 170 | str(err.exception), 171 | "Can't build ExecutionResponse from {'error': 'invalid API Key'}", 172 | ) 173 | with self.assertRaises(DuneError) as err: 174 | dune.get_execution_status("wonky job_id") 175 | self.assertEqual( 176 | str(err.exception), 177 | "Can't build ExecutionStatusResponse from {'error': 'invalid API Key'}", 178 | ) 179 | with self.assertRaises(DuneError) as err: 180 | dune.get_execution_results("wonky job_id") 181 | self.assertEqual( 182 | str(err.exception), 183 | "Can't build ResultsResponse from {'error': 'invalid API Key'}", 184 | ) 185 | 186 | def test_query_not_found_error(self): 187 | dune = DuneClient(self.valid_api_key) 188 | query = copy.copy(self.query) 189 | query.query_id = 99999999 # Invalid Query Id. 190 | 191 | with self.assertRaises(DuneError) as err: 192 | dune.execute_query(query) 193 | self.assertEqual( 194 | str(err.exception), 195 | "Can't build ExecutionResponse from {'error': 'Query not found'}", 196 | ) 197 | 198 | def test_internal_error(self): 199 | dune = DuneClient(self.valid_api_key) 200 | query = copy.copy(self.query) 201 | # This query ID is too large! 202 | query.query_id = 9999999999999 203 | 204 | with self.assertRaises(DuneError) as err: 205 | dune.execute_query(query) 206 | self.assertEqual( 207 | str(err.exception), 208 | "Can't build ExecutionResponse from {'error': 'An internal error occured'}", 209 | ) 210 | 211 | def test_invalid_job_id_error(self): 212 | dune = DuneClient(self.valid_api_key) 213 | with self.assertRaises(DuneError) as err: 214 | dune.get_execution_status("Wonky Job ID") 215 | self.assertEqual( 216 | str(err.exception), 217 | "Can't build ExecutionStatusResponse from " 218 | "{'error': 'The requested execution ID (ID: Wonky Job ID) is invalid.'}", 219 | ) 220 | 221 | def test_get_latest_result_with_query_object(self): 222 | dune = DuneClient(self.valid_api_key) 223 | results = dune.get_latest_result(self.query).get_rows() 224 | self.assertGreater(len(results), 0) 225 | 226 | def test_get_latest_result_with_query_id(self): 227 | dune = DuneClient(self.valid_api_key) 228 | results = dune.get_latest_result(self.query.query_id).get_rows() 229 | self.assertGreater(len(results), 0) 230 | 231 | @unittest.skip("Requires custom namespace and table_name input.") 232 | def test_upload_csv_success(self): 233 | client = DuneClient(self.valid_api_key) 234 | self.assertEqual( 235 | client.upload_csv( 236 | table_name="e2e-test", 237 | description="best data", 238 | data="column1,column2\nvalue1,value2\nvalue3,value4", 239 | ), 240 | True, 241 | ) 242 | 243 | @unittest.skip("Requires custom namespace and table_name input.") 244 | def test_create_table_success(self): 245 | # Make sure the table doesn't already exist. 246 | # You will need to change the namespace to your own. 247 | client = DuneClient(self.valid_api_key) 248 | 249 | namespace = "bh2smith" 250 | table_name = "dataset_e2e_test" 251 | 252 | self.assertEqual( 253 | client.create_table( 254 | namespace=namespace, 255 | table_name=table_name, 256 | description="e2e test table", 257 | schema=[ 258 | {"name": "date", "type": "timestamp"}, 259 | {"name": "dgs10", "type": "double"}, 260 | ], 261 | is_private=False, 262 | ), 263 | CreateTableResult.from_dict( 264 | { 265 | "namespace": namespace, 266 | "table_name": table_name, 267 | "full_name": f"dune.{namespace}.{table_name}", 268 | "example_query": f"select * from dune.{namespace}.{table_name} limit 10", 269 | } 270 | ), 271 | ) 272 | 273 | # @unittest.skip("Requires custom namespace and table_name input.") 274 | def test_create_table_error(self): 275 | client = DuneClient("Invalid Key") 276 | 277 | namespace = "test" 278 | table_name = "table" 279 | with self.assertRaises(DuneError) as err: 280 | client.create_table( 281 | namespace=namespace, 282 | table_name=table_name, 283 | description="", 284 | schema=[ 285 | {"name": "ALL_CAPS", "type": "timestamp"}, 286 | ], 287 | is_private=False, 288 | ) 289 | 290 | @unittest.skip("Requires custom namespace and table_name input.") 291 | def test_insert_table_csv_success(self): 292 | # Make sure the table already exists and csv matches table schema. 293 | # You will need to change the namespace to your own. 294 | client = DuneClient(self.valid_api_key) 295 | namespace = "bh2smith" 296 | table_name = "dataset_e2e_test" 297 | client.create_table( 298 | namespace, 299 | table_name, 300 | schema=[ 301 | {"name": "date", "type": "timestamp"}, 302 | {"name": "dgs10", "type": "double"}, 303 | ], 304 | ) 305 | with open("./tests/fixtures/sample_table_insert.csv", "rb") as data: 306 | self.assertEqual( 307 | client.insert_table( 308 | namespace, 309 | table_name, 310 | data=data, 311 | content_type="text/csv", 312 | ), 313 | InsertTableResult(rows_written=1, bytes_written=33), 314 | ) 315 | 316 | @unittest.skip("Requires custom namespace and table_name input.") 317 | def test_clear_data(self): 318 | client = DuneClient(self.valid_api_key) 319 | namespace = "bh2smith" 320 | table_name = "dataset_e2e_test" 321 | self.assertEqual( 322 | client.clear_data(namespace, table_name), 323 | ClearTableResult( 324 | message="Table dune.bh2smith.dataset_e2e_test successfully cleared" 325 | ), 326 | ) 327 | 328 | @unittest.skip("Requires custom namespace and table_name input.") 329 | def test_insert_table_json_success(self): 330 | # Make sure the table already exists and json matches table schema. 331 | # You will need to change the namespace to your own. 332 | client = DuneClient(self.valid_api_key) 333 | with open("./tests/fixtures/sample_table_insert.json", "rb") as data: 334 | self.assertEqual( 335 | client.insert_table( 336 | namespace="test", 337 | table_name="dataset_e2e_test", 338 | data=data, 339 | content_type="application/x-ndjson", 340 | ), 341 | InsertTableResult(rows_written=1, bytes_written=33), 342 | ) 343 | 344 | @unittest.skip("Requires custom namespace and table_name input.") 345 | def test_delete_table_success(self): 346 | # Make sure the table doesn't already exist. 347 | # You will need to change the namespace to your own. 348 | client = DuneClient(self.valid_api_key) 349 | 350 | namespace = "test" 351 | table_name = "dataset_e2e_test" 352 | 353 | self.assertEqual( 354 | client.delete_table( 355 | namespace=namespace, 356 | table_name=table_name, 357 | ), 358 | DeleteTableResult.from_dict( 359 | { 360 | "message": "Table teamwaddah.waddah_test3 successfully deleted", 361 | } 362 | ), 363 | ) 364 | 365 | def test_download_csv_with_pagination(self): 366 | # Arrange 367 | client = DuneClient(self.valid_api_key) 368 | client.run_query(self.multi_rows_query) 369 | 370 | # Act 371 | result_csv = client.download_csv(self.multi_rows_query.query_id, batch_size=1) 372 | 373 | # Assert 374 | self.assertEqual( 375 | pandas.read_csv(result_csv.data).to_dict(orient="records"), 376 | [ 377 | {"number": 1}, 378 | {"number": 2}, 379 | {"number": 3}, 380 | {"number": 4}, 381 | {"number": 5}, 382 | ], 383 | ) 384 | 385 | def test_download_csv_with_filters(self): 386 | # Arrange 387 | client = DuneClient(self.valid_api_key) 388 | client.run_query(self.multi_rows_query) 389 | 390 | # Act 391 | result_csv = client.download_csv( 392 | self.multi_rows_query.query_id, 393 | filters="number < 3", 394 | ) 395 | 396 | # Assert 397 | self.assertEqual( 398 | pandas.read_csv(result_csv.data).to_dict(orient="records"), 399 | [ 400 | {"number": 1}, 401 | {"number": 2}, 402 | ], 403 | ) 404 | 405 | def test_download_csv_success_by_id(self): 406 | client = DuneClient(self.valid_api_key) 407 | new_query = self.copy_query_and_change_parameters() 408 | # Run query with new parameters 409 | client.run_query(new_query) 410 | # Download CSV by query_id 411 | result_csv = client.download_csv(self.query.query_id) 412 | # Expect that the csv returns the latest execution results (i.e. those that were just run) 413 | self.assertEqual( 414 | pandas.read_csv(result_csv.data).to_dict(orient="records"), 415 | [ 416 | { 417 | "text_field": "different word", 418 | "number_field": 22, 419 | "date_field": "1991-01-01 00:00:00", 420 | "list_field": "Option 2", 421 | } 422 | ], 423 | ) 424 | 425 | def test_download_csv_success_with_params(self): 426 | client = DuneClient(self.valid_api_key) 427 | # Download CSV with query and given parameters. 428 | result_csv = client.download_csv(self.query) 429 | # Expect the result to be relative to values of given parameters. 430 | ################################################################# 431 | # Note that we could compare results with 432 | # ",".join([p.value for p in self.query.parameters()]) + "\n" 433 | # but there seems to be a discrepancy with the date string values. 434 | # Specifically 1991-01-01 00:00:00 435 | # vs 1991-01-01 00:00:00 436 | ################################################################# 437 | self.assertEqual( 438 | pandas.read_csv(result_csv.data).to_dict(orient="records"), 439 | [ 440 | { 441 | "date_field": "2022-05-04 00:00:00", 442 | "list_field": "Option 1", 443 | "number_field": 3.1415926535, 444 | "text_field": "Plain Text", 445 | } 446 | ], 447 | ) 448 | 449 | 450 | @unittest.skip("This is an enterprise only endpoint that can no longer be tested.") 451 | class TestCRUDOps(unittest.TestCase): 452 | def setUp(self) -> None: 453 | self.valid_api_key = os.environ["DUNE_API_KEY"] 454 | self.client = DuneClient(self.valid_api_key, client_version="alpha/v1") 455 | self.existing_query_id = 2713571 456 | 457 | @unittest.skip("Works fine, but creates too many queries") 458 | def test_create(self): 459 | new_query = self.client.create_query(name="test_create", query_sql="") 460 | self.assertGreater(new_query.base.query_id, 0) 461 | 462 | def test_get(self): 463 | q_id = 12345 464 | query = self.client.get_query(q_id) 465 | self.assertEqual(query.base.query_id, q_id) 466 | 467 | def test_update(self): 468 | test_id = self.existing_query_id 469 | current_sql = self.client.get_query(test_id).sql 470 | self.client.update_query(query_id=test_id, query_sql="") 471 | self.assertEqual(self.client.get_query(test_id).sql, "") 472 | # Reset: 473 | self.client.update_query(query_id=test_id, query_sql=current_sql) 474 | 475 | def test_make_private_and_public(self): 476 | q_id = self.existing_query_id 477 | self.client.make_private(q_id) 478 | self.assertEqual(self.client.get_query(q_id).meta.is_private, True) 479 | self.client.make_public(q_id) 480 | self.assertEqual(self.client.get_query(q_id).meta.is_private, False) 481 | 482 | def test_archive(self): 483 | self.assertEqual(self.client.archive_query(self.existing_query_id), True) 484 | self.assertEqual(self.client.unarchive_query(self.existing_query_id), False) 485 | 486 | @unittest.skip("Works fine, but creates too many queries!") 487 | def test_run_sql(self): 488 | query_sql = "select 85" 489 | results = self.client.run_sql(query_sql) 490 | self.assertEqual(results.get_rows(), [{"_col0": 85}]) 491 | 492 | # The default functionality is meant to create a private query and then archive it. 493 | query = self.client.get_query(results.query_id) 494 | self.assertTrue(query.meta.is_archived) 495 | self.assertTrue(query.meta.is_private) 496 | 497 | 498 | if __name__ == "__main__": 499 | unittest.main() 500 | -------------------------------------------------------------------------------- /tests/e2e/test_custom_endpoints.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import os 3 | import time 4 | import unittest 5 | 6 | import dotenv 7 | 8 | from dune_client.client import DuneClient 9 | 10 | dotenv.load_dotenv() 11 | 12 | 13 | @unittest.skip("endpoint no longer exists - {'error': 'Custom endpoint not found'}") 14 | class TestCustomEndpoints(unittest.TestCase): 15 | def setUp(self) -> None: 16 | self.valid_api_key = os.environ["DUNE_API_KEY"] 17 | 18 | def test_getting_custom_endpoint_results(self): 19 | dune = DuneClient(self.valid_api_key) 20 | results = dune.get_custom_endpoint_result("dune", "new-test") 21 | self.assertEqual(len(results.get_rows()), 10) 22 | 23 | 24 | if __name__ == "__main__": 25 | unittest.main() 26 | -------------------------------------------------------------------------------- /tests/fixtures/sample_table_insert.csv: -------------------------------------------------------------------------------- 1 | date,dgs10 2 | 2020-12-01T23:33:00,10 -------------------------------------------------------------------------------- /tests/fixtures/sample_table_insert.json: -------------------------------------------------------------------------------- 1 | {"date":"2020-12-01T23:33:00","dgs10":10} -------------------------------------------------------------------------------- /tests/unit/test_file.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import unittest 4 | 5 | from dune_client.file.base import CSVFile, NDJSONFile, JSONFile 6 | from dune_client.file.interface import FileIO 7 | 8 | TEST_FILE = "test" 9 | TEST_PATH = "tmp" 10 | 11 | FILE_WRITERS = [ 12 | CSVFile(TEST_PATH, TEST_FILE + ".csv"), 13 | NDJSONFile(TEST_PATH, TEST_FILE + ".ndjson"), 14 | JSONFile(TEST_PATH, TEST_FILE + ".json"), 15 | ] 16 | 17 | 18 | def cleanup(): 19 | for writer in FILE_WRITERS: 20 | try: 21 | os.remove(writer.filepath) 22 | except FileNotFoundError: 23 | pass 24 | 25 | 26 | def cleanup_files(func): 27 | """This decorator can be used for testing methods outside this class""" 28 | 29 | def wrapped_func(self): 30 | func(self) 31 | cleanup() 32 | 33 | return wrapped_func 34 | 35 | 36 | class TestFileIO(unittest.TestCase): 37 | """These tests indirectly test FileType's read and write functionality.""" 38 | 39 | def setUp(self) -> None: 40 | self.dune_records = [ 41 | {"col1": "value01", "col2": "value02"}, 42 | {"col1": "value11", "col2": "value12"}, 43 | ] 44 | self.file_manager = FileIO(TEST_PATH) 45 | self.file_writers = FILE_WRITERS 46 | 47 | def tearDown(self) -> None: 48 | cleanup() 49 | 50 | def test_invertible_write_and_load(self): 51 | for writer in self.file_writers: 52 | self.file_manager._write(self.dune_records, writer, True) 53 | loaded_records = self.file_manager._load(writer) 54 | self.assertEqual( 55 | self.dune_records, loaded_records, f"test invertible failed on {writer}" 56 | ) 57 | 58 | def test_append_ok(self): 59 | for writer in self.file_writers: 60 | self.file_manager._write(self.dune_records, writer, True) 61 | self.file_manager._append(self.dune_records, writer, True) 62 | loaded_records = self.file_manager._load(writer) 63 | expected = self.dune_records + self.dune_records 64 | self.assertEqual(expected, loaded_records, f"append failed on {writer}") 65 | 66 | def test_append_calls_write_on_new_file(self): 67 | for writer in self.file_writers: 68 | with self.assertLogs(level="WARNING"): 69 | self.file_manager._append(self.dune_records, writer, True) 70 | 71 | def test_append_error(self): 72 | invalid_records = [{}] # Empty dict has different keys than self.dune_records 73 | for writer in self.file_writers: 74 | self.file_manager._write(self.dune_records, writer, True) 75 | with self.assertRaises(AssertionError): 76 | self.file_manager._append(invalid_records, writer, True) 77 | 78 | def test_load_singleton(self): 79 | for writer in self.file_writers: 80 | self.file_manager._write(self.dune_records, writer, True) 81 | entry_0 = self.file_manager.load_singleton( 82 | name="Doesn't matter is in the writer", ftype=writer 83 | ) 84 | entry_1 = self.file_manager.load_singleton( 85 | "Doesn't matter is in the writer", writer, 1 86 | ) 87 | self.assertEqual( 88 | self.dune_records[0], entry_0, f"failed on {writer} at index 0" 89 | ) 90 | self.assertEqual( 91 | self.dune_records[1], entry_1, f"failed on {writer} at index 1" 92 | ) 93 | 94 | for extension in [".csv", ".json", ".ndjson"]: 95 | # Files were already written above. 96 | entry_0 = self.file_manager.load_singleton(TEST_FILE + extension, extension) 97 | entry_1 = self.file_manager.load_singleton( 98 | TEST_FILE + extension, extension, 1 99 | ) 100 | self.assertEqual( 101 | self.dune_records[0], 102 | entry_0, 103 | f"failed on {extension} (extension) at index 0", 104 | ) 105 | self.assertEqual( 106 | self.dune_records[1], 107 | entry_1, 108 | f"failed on {extension} (extension) at index 1", 109 | ) 110 | 111 | def test_write_any_format_with_arbitrary_extension(self): 112 | weird_name = "weird_file.ext" 113 | weird_files = [ 114 | NDJSONFile(TEST_PATH, weird_name), 115 | JSONFile(TEST_PATH, weird_name), 116 | CSVFile(TEST_PATH, weird_name), 117 | ] 118 | extensions = [ 119 | "ndjson", 120 | "json", 121 | "csv", 122 | ] 123 | for weird_file, ext in zip(weird_files, extensions): 124 | self.file_manager._write(self.dune_records, weird_file, True) 125 | self.file_manager._load(weird_file) 126 | entry_0 = self.file_manager.load_singleton(weird_name, ext) 127 | entry_1 = self.file_manager.load_singleton( 128 | "meaningless string", weird_file, 1 129 | ) 130 | self.assertEqual( 131 | self.dune_records[0], 132 | entry_0, 133 | f"failed on {weird_file} at index 0", 134 | ) 135 | self.assertEqual( 136 | self.dune_records[1], 137 | entry_1, 138 | f"failed on {weird_file} at index 1", 139 | ) 140 | 141 | def test_skip_empty_write(self): 142 | for writer in self.file_writers: 143 | with self.assertLogs(): 144 | self.file_manager._write([], writer, True) 145 | with self.assertRaises(FileNotFoundError): 146 | self.file_manager._load(writer) 147 | 148 | def test_not_skip_empty_when_specified(self): 149 | for writer in self.file_writers: 150 | if isinstance(writer, CSVFile): 151 | with self.assertLogs(level="WARNING"): 152 | # CSV empty files won't have any headers! 153 | self.file_manager._write([], writer, False) 154 | else: 155 | if sys.version_info < (3, 10): 156 | with self.assertRaises(FileNotFoundError): 157 | self.file_manager._load(writer) 158 | # assertNoLogs didn't exist till python 3.10, but we still support lower versions. 159 | # This is a bit of a hack, we write and then load to ensure the empty file was written. 160 | self.file_manager._write([], writer, False) 161 | # _load would return FileNotFoundError if it hadn't been written 162 | self.assertEqual(0, len(self.file_manager._load(writer))) 163 | else: 164 | with self.assertNoLogs(): 165 | self.file_manager._write([], writer, False) 166 | 167 | self.file_manager._load(writer) 168 | 169 | def test_idempotent_write(self): 170 | for writer in self.file_writers: 171 | self.file_manager._write(self.dune_records, writer, True) 172 | self.file_manager._write(self.dune_records, writer, True) 173 | self.assertEqual( 174 | self.dune_records, 175 | self.file_manager._load(writer), 176 | f"idempotent write failed on {writer}", 177 | ) 178 | -------------------------------------------------------------------------------- /tests/unit/test_models.py: -------------------------------------------------------------------------------- 1 | import json 2 | import unittest 3 | import csv 4 | from datetime import datetime 5 | from io import BytesIO, TextIOWrapper 6 | 7 | from dateutil.parser import parse 8 | from dateutil.tz import tzutc 9 | 10 | from dune_client.models import ( 11 | ExecutionResponse, 12 | ExecutionResultCSV, 13 | ExecutionStatusResponse, 14 | ExecutionState, 15 | ResultsResponse, 16 | TimeData, 17 | ExecutionResult, 18 | ResultMetadata, 19 | DuneError, 20 | ) 21 | from dune_client.types import QueryParameter 22 | from dune_client.query import DuneQuery, QueryMeta, QueryBase 23 | 24 | 25 | class MyTestCase(unittest.TestCase): 26 | def setUp(self) -> None: 27 | self.execution_id = "01GBM4W2N0NMCGPZYW8AYK4YF1" 28 | self.query_id = 980708 29 | self.submission_time_str = "2022-08-29T06:33:24.913138Z" 30 | self.execution_start_str = "2022-08-29T06:33:24.916543331Z" 31 | self.execution_end_str = "2022-08-29T06:33:25.816543331Z" 32 | 33 | self.execution_response_data = { 34 | "execution_id": self.execution_id, 35 | "state": "QUERY_STATE_PENDING", 36 | } 37 | self.status_response_data = { 38 | "execution_id": self.execution_id, 39 | "query_id": self.query_id, 40 | "state": "QUERY_STATE_EXECUTING", 41 | "submitted_at": self.submission_time_str, 42 | "execution_started_at": self.execution_start_str, 43 | "execution_ended_at": self.execution_end_str, 44 | } 45 | self.result_metadata_data = { 46 | "column_names": ["ct", "TableName"], 47 | "column_types": ["x", "y"], 48 | "row_count": 8, 49 | "result_set_bytes": 194, 50 | "total_result_set_bytes": 194, 51 | "total_row_count": 8, 52 | "datapoint_count": 2, 53 | "pending_time_millis": 54, 54 | "execution_time_millis": 900, 55 | } 56 | self.status_response_data_completed = { 57 | "execution_id": self.execution_id, 58 | "query_id": self.query_id, 59 | "state": "QUERY_STATE_COMPLETED", 60 | "submitted_at": self.submission_time_str, 61 | "execution_started_at": self.execution_start_str, 62 | "execution_ended_at": self.execution_end_str, 63 | "result_metadata": self.result_metadata_data, 64 | } 65 | self.results_response_data = { 66 | "execution_id": self.execution_id, 67 | "query_id": self.query_id, 68 | "state": "QUERY_STATE_COMPLETED", 69 | "submitted_at": self.submission_time_str, 70 | "expires_at": "2024-08-28T06:36:41.58847Z", 71 | "execution_started_at": self.execution_start_str, 72 | "execution_ended_at": self.execution_end_str, 73 | "result": { 74 | "rows": [ 75 | {"TableName": "eth_blocks", "ct": 6296}, 76 | {"TableName": "eth_traces", "ct": 4474223}, 77 | ], 78 | "metadata": self.result_metadata_data, 79 | }, 80 | } 81 | self.execution_result_csv_data = BytesIO( 82 | b"""TableName,ct 83 | eth_blocks,6296 84 | eth_traces,4474223 85 | """, 86 | ) 87 | 88 | def test_execution_response_parsing(self): 89 | expected = ExecutionResponse( 90 | execution_id="01GBM4W2N0NMCGPZYW8AYK4YF1", 91 | state=ExecutionState.PENDING, 92 | ) 93 | 94 | self.assertEqual( 95 | expected, ExecutionResponse.from_dict(self.execution_response_data) 96 | ) 97 | 98 | def test_parse_time_data(self): 99 | expected_with_end = TimeData( 100 | submitted_at=parse(self.submission_time_str), 101 | expires_at=datetime(2024, 8, 28, 6, 36, 41, 588470, tzinfo=tzutc()), 102 | execution_started_at=parse(self.execution_start_str), 103 | execution_ended_at=parse(self.execution_end_str), 104 | cancelled_at=None, 105 | ) 106 | self.assertEqual( 107 | expected_with_end, TimeData.from_dict(self.results_response_data) 108 | ) 109 | 110 | expected_with_empty_optionals = TimeData( 111 | submitted_at=parse(self.submission_time_str), 112 | expires_at=None, 113 | execution_started_at=parse(self.execution_start_str), 114 | execution_ended_at=parse(self.execution_end_str), 115 | cancelled_at=None, 116 | ) 117 | self.assertEqual( 118 | expected_with_empty_optionals, TimeData.from_dict(self.status_response_data) 119 | ) 120 | 121 | def test_parse_status_response(self): 122 | expected = ExecutionStatusResponse( 123 | execution_id="01GBM4W2N0NMCGPZYW8AYK4YF1", 124 | query_id=980708, 125 | state=ExecutionState.EXECUTING, 126 | times=TimeData.from_dict(self.status_response_data), 127 | result_metadata=None, 128 | queue_position=None, 129 | error=None, 130 | ) 131 | self.assertEqual( 132 | expected, ExecutionStatusResponse.from_dict(self.status_response_data) 133 | ) 134 | 135 | def test_parse_known_status_response(self): 136 | # For context: https://github.com/cowprotocol/dune-client/issues/22 137 | response = { 138 | "execution_id": "01GES18035K5C4GDTY12Q79GBD", 139 | "query_id": 1317323, 140 | "state": "QUERY_STATE_COMPLETED", 141 | "submitted_at": "2022-10-07T10:53:18.822127Z", 142 | "expires_at": "2024-10-06T10:53:20.729373Z", 143 | "execution_started_at": "2022-10-07T10:53:18.823105936Z", 144 | "execution_ended_at": "2022-10-07T10:53:20.729372559Z", 145 | "result_metadata": { 146 | "column_names": ["token"], 147 | "column_types": ["varchar"], 148 | "result_set_bytes": 815, 149 | "total_row_count": 18, 150 | "datapoint_count": 18, 151 | "execution_time_millis": 1906, 152 | }, 153 | } 154 | try: 155 | ExecutionStatusResponse.from_dict(response) 156 | except DuneError as err: 157 | self.fail(f"Unexpected error {err}") 158 | 159 | def test_parse_status_response_completed(self): 160 | self.assertEqual( 161 | ExecutionStatusResponse( 162 | execution_id="01GBM4W2N0NMCGPZYW8AYK4YF1", 163 | query_id=980708, 164 | state=ExecutionState.COMPLETED, 165 | times=TimeData.from_dict(self.status_response_data), 166 | result_metadata=ResultMetadata.from_dict(self.result_metadata_data), 167 | queue_position=None, 168 | error=None, 169 | ), 170 | ExecutionStatusResponse.from_dict(self.status_response_data_completed), 171 | ) 172 | 173 | def test_parse_result_metadata(self): 174 | expected = ResultMetadata( 175 | column_names=["ct", "TableName"], 176 | column_types=["x", "y"], 177 | row_count=8, 178 | result_set_bytes=194, 179 | total_row_count=8, 180 | total_result_set_bytes=194, 181 | datapoint_count=2, 182 | pending_time_millis=54, 183 | execution_time_millis=900, 184 | ) 185 | self.assertEqual( 186 | expected, 187 | ResultMetadata.from_dict(self.results_response_data["result"]["metadata"]), 188 | ) 189 | self.assertEqual( 190 | expected, 191 | ResultMetadata.from_dict( 192 | self.status_response_data_completed["result_metadata"] 193 | ), 194 | ) 195 | 196 | def test_parse_execution_result(self): 197 | expected = ExecutionResult( 198 | rows=[ 199 | {"TableName": "eth_blocks", "ct": 6296}, 200 | {"TableName": "eth_traces", "ct": 4474223}, 201 | ], 202 | # Parsing tested above in test_result_metadata_parsing 203 | metadata=ResultMetadata.from_dict( 204 | self.results_response_data["result"]["metadata"] 205 | ), 206 | ) 207 | 208 | self.assertEqual( 209 | expected, ExecutionResult.from_dict(self.results_response_data["result"]) 210 | ) 211 | 212 | def test_parse_result_response(self): 213 | # Time data parsing tested above in test_time_data_parsing. 214 | time_data = TimeData.from_dict(self.results_response_data) 215 | expected = ResultsResponse( 216 | execution_id=self.execution_id, 217 | query_id=self.query_id, 218 | state=ExecutionState.COMPLETED, 219 | times=time_data, 220 | # Execution result parsing tested above in test_execution_result 221 | result=ExecutionResult.from_dict(self.results_response_data["result"]), 222 | next_uri=None, 223 | next_offset=None, 224 | ) 225 | self.assertEqual( 226 | expected, ResultsResponse.from_dict(self.results_response_data) 227 | ) 228 | 229 | def test_execution_result_csv(self): 230 | # document the expected output data from DuneAPI result/csv endpoint 231 | csv_response = ExecutionResultCSV(data=self.execution_result_csv_data) 232 | result = csv.reader(TextIOWrapper(csv_response.data)) 233 | # note that CSV is non-typed, up to the reader to do type inference 234 | self.assertEqual( 235 | [ 236 | ["TableName", "ct"], 237 | ["eth_blocks", "6296"], 238 | ["eth_traces", "4474223"], 239 | ], 240 | [r for r in result], 241 | ) 242 | 243 | def test_dune_query_from_dict(self): 244 | example_response = """{ 245 | "query_id": 60066, 246 | "name": "Ethereum transactions", 247 | "description": "Returns ethereum transactions starting from the oldest by block time", 248 | "tags": ["ethereum", "transactions"], 249 | "version": 15, 250 | "parameters": [{"key": "limit", "value": "5", "type": "number"}], 251 | "query_engine": "v2 Dune SQL", 252 | "query_sql": "select block_number from ethereum.transactions limit {{limit}};", 253 | "is_private": true, 254 | "is_archived": false, 255 | "is_unsaved": false, 256 | "owner": "Owner Name" 257 | }""" 258 | expected = DuneQuery( 259 | base=QueryBase( 260 | query_id=60066, 261 | name="Ethereum transactions", 262 | params=[ 263 | QueryParameter.from_dict( 264 | {"key": "limit", "value": "5", "type": "number"} 265 | ) 266 | ], 267 | ), 268 | meta=QueryMeta( 269 | description="Returns ethereum transactions starting from the oldest by block time", 270 | tags=["ethereum", "transactions"], 271 | version=15, 272 | engine="v2 Dune SQL", 273 | is_private=True, 274 | is_archived=False, 275 | is_unsaved=False, 276 | owner="Owner Name", 277 | ), 278 | sql="select block_number from ethereum.transactions limit {{limit}};", 279 | ) 280 | self.assertEqual(expected, DuneQuery.from_dict(json.loads(example_response))) 281 | 282 | 283 | if __name__ == "__main__": 284 | unittest.main() 285 | -------------------------------------------------------------------------------- /tests/unit/test_query.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from datetime import datetime 3 | 4 | from dune_client.query import QueryBase, parse_query_object_or_id 5 | from dune_client.types import QueryParameter 6 | 7 | 8 | class TestQueryBase(unittest.TestCase): 9 | def setUp(self) -> None: 10 | self.date = datetime(year=1985, month=3, day=10) 11 | self.query_params = [ 12 | QueryParameter.enum_type("Enum", "option1"), 13 | QueryParameter.text_type("Text", "plain text"), 14 | QueryParameter.number_type("Number", 12), 15 | QueryParameter.date_type("Date", "2021-01-01 12:34:56"), 16 | ] 17 | self.query = QueryBase(name="", query_id=0, params=self.query_params) 18 | 19 | def test_base_url(self): 20 | self.assertEqual(self.query.base_url(), "https://dune.com/queries/0") 21 | 22 | def test_url(self): 23 | self.assertEqual( 24 | self.query.url(), 25 | "https://dune.com/queries/0?Enum=option1&Text=plain+text&Number=12&Date=2021-01-01+12%3A34%3A56", 26 | ) 27 | self.assertEqual(QueryBase(0, "", []).url(), "https://dune.com/queries/0") 28 | 29 | def test_parameters(self): 30 | self.assertEqual(self.query.parameters(), self.query_params) 31 | 32 | def test_request_format(self): 33 | expected_answer = { 34 | "query_parameters": { 35 | "Enum": "option1", 36 | "Text": "plain text", 37 | "Number": "12", 38 | "Date": "2021-01-01 12:34:56", 39 | } 40 | } 41 | self.assertEqual(self.query.request_format(), expected_answer) 42 | 43 | def test_hash(self): 44 | # Same ID, different params 45 | query1 = QueryBase( 46 | query_id=0, params=[QueryParameter.text_type("Text", "word1")] 47 | ) 48 | query2 = QueryBase( 49 | query_id=0, params=[QueryParameter.text_type("Text", "word2")] 50 | ) 51 | self.assertNotEqual(hash(query1), hash(query2)) 52 | 53 | # Different ID, same 54 | query1 = QueryBase(query_id=0) 55 | query2 = QueryBase(query_id=1) 56 | self.assertNotEqual(hash(query1), hash(query2)) 57 | 58 | # Different ID different params 59 | query1 = QueryBase(query_id=0) 60 | query2 = QueryBase(query_id=1, params=[QueryParameter.number_type("num", 1)]) 61 | self.assertNotEqual(hash(query1), hash(query2)) 62 | 63 | def test_parse_object_or_id(self): 64 | expected_params = { 65 | "params.Date": "2021-01-01 12:34:56", 66 | "params.Enum": "option1", 67 | "params.Number": "12", 68 | "params.Text": "plain text", 69 | } 70 | expected_query_id = self.query.query_id 71 | # Query Object 72 | self.assertEqual( 73 | parse_query_object_or_id(self.query), (expected_params, expected_query_id) 74 | ) 75 | # Query ID (integer) 76 | expected_params = None 77 | self.assertEqual( 78 | parse_query_object_or_id(self.query.query_id), 79 | (expected_params, expected_query_id), 80 | ) 81 | # Query ID (string) 82 | self.assertEqual( 83 | parse_query_object_or_id(str(self.query.query_id)), 84 | (expected_params, expected_query_id), 85 | ) 86 | 87 | 88 | if __name__ == "__main__": 89 | unittest.main() 90 | -------------------------------------------------------------------------------- /tests/unit/test_types.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import unittest 3 | 4 | from dune_client.query import QueryBase 5 | from dune_client.types import QueryParameter, Address 6 | 7 | 8 | class TestAddress(unittest.TestCase): 9 | def setUp(self) -> None: 10 | self.lower_case_address = "0xde1c59bc25d806ad9ddcbe246c4b5e5505645718" 11 | self.check_sum_address = "0xDEf1CA1fb7FBcDC777520aa7f396b4E015F497aB" 12 | self.invalid_address = "0x12" 13 | self.dune_format = "\\x5d4020b9261f01b6f8a45db929704b0ad6f5e9e6" 14 | 15 | def test_invalid(self): 16 | with self.assertRaises(ValueError) as err: 17 | Address(address=self.invalid_address) 18 | self.assertEqual( 19 | str(err.exception), f"Invalid Ethereum Address {self.invalid_address}" 20 | ) 21 | 22 | def test_valid(self): 23 | self.assertEqual( 24 | Address(address=self.lower_case_address).address, 25 | "0xde1c59bc25d806ad9ddcbe246c4b5e5505645718", 26 | ) 27 | self.assertEqual( 28 | Address(address=self.check_sum_address).address, 29 | "0xdef1ca1fb7fbcdc777520aa7f396b4e015f497ab", 30 | ) 31 | self.assertEqual( 32 | Address(address=self.dune_format).address, 33 | "0x5d4020b9261f01b6f8a45db929704b0ad6f5e9e6", 34 | ) 35 | 36 | def test_inequality(self): 37 | zero = Address("0x0000000000000000000000000000000000000000") 38 | one = Address("0x1000000000000000000000000000000000000000") 39 | a_lower = Address("0xa000000000000000000000000000000000000000") 40 | a_upper = Address("0xA000000000000000000000000000000000000000") 41 | b_lower = Address("0xb000000000000000000000000000000000000000") 42 | c_upper = Address("0xC000000000000000000000000000000000000000") 43 | self.assertLess(zero, one) 44 | self.assertLess(one, a_lower) 45 | self.assertEqual(a_lower, a_upper) 46 | self.assertLess(a_upper, b_lower) 47 | self.assertLess(b_lower, c_upper) 48 | 49 | 50 | class TestQueryParameter(unittest.TestCase): 51 | def setUp(self) -> None: 52 | self.number_type = QueryParameter.number_type("Number", 1) 53 | self.text_type = QueryParameter.text_type("Text", "hello") 54 | self.date_type = QueryParameter.date_type( 55 | "Date", datetime.datetime(2022, 3, 10) 56 | ) 57 | 58 | def test_constructors_and_to_dict(self): 59 | self.assertEqual( 60 | self.number_type.to_dict(), 61 | {"key": "Number", "type": "number", "value": "1"}, 62 | ) 63 | self.assertEqual( 64 | self.text_type.to_dict(), {"key": "Text", "type": "text", "value": "hello"} 65 | ) 66 | self.assertEqual( 67 | self.date_type.to_dict(), 68 | {"key": "Date", "type": "datetime", "value": "2022-03-10 00:00:00"}, 69 | ) 70 | 71 | def test_repr_method(self): 72 | query = QueryBase( 73 | query_id=1, 74 | name="Test Query", 75 | params=[self.number_type, self.text_type], 76 | ) 77 | 78 | self.assertEqual( 79 | "QueryBase(query_id=1, name='Test Query', " 80 | "params=[" 81 | "Parameter(name=Number, value=1, type=number), " 82 | "Parameter(name=Text, value=hello, type=text)" 83 | "])", 84 | str(query), 85 | ) 86 | 87 | 88 | if __name__ == "__main__": 89 | unittest.main() 90 | -------------------------------------------------------------------------------- /tests/unit/test_utils.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import unittest 3 | from dune_client.util import get_package_version, age_in_hours 4 | 5 | 6 | class TestUtils(unittest.TestCase): 7 | def test_package_version_some(self): 8 | version_string = get_package_version("requests") 9 | parsed_version = list(map(int, version_string.split("."))) 10 | self.assertGreaterEqual(parsed_version, [2, 31, 0]) 11 | 12 | def test_package_version_none(self): 13 | # Can't self refer (this should only work when user has dune-client installed). 14 | self.assertIsNone(get_package_version("unittest")) 15 | 16 | def test_age_in_hours(self): 17 | march_ten_eighty_five = datetime.datetime( 18 | 1985, 3, 10, tzinfo=datetime.timezone.utc 19 | ) 20 | self.assertGreaterEqual(age_in_hours(march_ten_eighty_five), 314159) 21 | -------------------------------------------------------------------------------- /tests/unit/test_viz_sankey.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest.mock import patch 3 | import pandas as pd 4 | from dune_client.viz.graphs import create_sankey 5 | 6 | 7 | class TestCreateSankey(unittest.TestCase): 8 | # Setting up a dataframe for testing 9 | def setUp(self): 10 | self.df = pd.DataFrame( 11 | { 12 | "source_col": ["WBTC", "USDC"], 13 | "target_col": ["USDC", "WBTC"], 14 | "value_col": [2184, 2076], 15 | } 16 | ) 17 | 18 | self.predefined_colors = { 19 | "USDC": "rgb(38, 112, 196)", 20 | "WBTC": "rgb(247, 150, 38)", 21 | } 22 | 23 | self.columns = { 24 | "source": "source_col", 25 | "target": "target_col", 26 | "value": "value_col", 27 | } 28 | self.viz_config: dict = { 29 | "node_pad": 15, 30 | "node_thickness": 20, 31 | "node_line_width": 0.5, 32 | "font_size": 10, 33 | "figure_height": 1000, 34 | "figure_width": 1500, 35 | } 36 | 37 | def test_missing_column(self): 38 | # Remove a required column from dataframe 39 | df_without_target = self.df.drop(columns=["target_col"]) 40 | with self.assertRaises(ValueError): 41 | create_sankey( 42 | df_without_target, self.predefined_colors, self.columns, self.viz_config 43 | ) 44 | 45 | def test_value_column_not_numeric(self): 46 | # Change the 'value' column to a non-numeric type 47 | df_with_str_values = self.df.copy() 48 | df_with_str_values["value_col"] = ["10", "11"] 49 | with self.assertRaises(ValueError): 50 | create_sankey( 51 | df_with_str_values, 52 | self.predefined_colors, 53 | self.columns, 54 | self.viz_config, 55 | ) 56 | 57 | # Mocking the visualization creation and just testing the processing logic 58 | @patch("plotly.graph_objects.Figure") 59 | def test_mocked_visualization(self, MockFigure): 60 | result = create_sankey( 61 | self.df, self.predefined_colors, self.columns, self.viz_config, "test" 62 | ) 63 | 64 | # Ensuring our mocked Figure was called with the correct parameters 65 | MockFigure.assert_called_once() 66 | 67 | 68 | if __name__ == "__main__": 69 | unittest.main() 70 | --------------------------------------------------------------------------------