├── .github └── workflows │ └── action-sqlfluff.yml ├── .gitignore ├── .sqlfluff ├── .sqlfluff-ci ├── LICENSE ├── README.md ├── assets └── dbt_datawaves_architecture.png ├── dbt_project.yml ├── integration_tests ├── dbt_project.yml ├── macros │ └── tests.sql ├── models │ └── utils │ │ ├── schema.yml │ │ ├── test_binary_to_address.sql │ │ ├── test_displayed_amount.sql │ │ ├── test_hex.sql │ │ └── test_substring.sql ├── packages.yml └── seeds │ └── utils │ ├── data_binary_to_address.csv │ ├── data_displayed_amount.csv │ ├── data_hex.csv │ └── data_substring.csv ├── macros ├── generate_schema_name.sql └── utils │ └── cte_import.sql ├── models ├── base │ └── base_ethereum_sources.yml ├── cryptopunks │ ├── cryptopunk_schema.yml │ ├── cryptopunks_trades.sql │ └── ethereum │ │ ├── cryptopunk_ethereum_schema.yml │ │ ├── cryptopunks_ethereum_sources.yml │ │ ├── cryptopunks_ethereum_trades.sql │ │ └── cryptopunks_ethereum_transfers.sql ├── ens │ ├── ens_registrations.sql │ ├── ens_reverse_registrars.sql │ ├── ens_schema.yml │ └── ethereum │ │ ├── ens_ethereum_registrations.sql │ │ ├── ens_ethereum_reverse_registrars.sql │ │ ├── ens_ethereum_schema.yml │ │ └── ens_ethereum_sources.yml ├── erc1155 │ └── ethereum │ │ ├── erc1155_ethereum_schema.yml │ │ ├── erc1155_ethereum_sources.yml │ │ └── erc1155_ethereum_transfers.sql ├── erc20 │ └── ethereum │ │ ├── erc20_ethereum_schema.yml │ │ ├── erc20_ethereum_sources.yml │ │ └── erc20_ethereum_transfers.sql ├── erc721 │ └── ethereum │ │ ├── erc721_ethereum_schema.yml │ │ ├── erc721_ethereum_sources.yml │ │ └── erc721_ethereum_transfers.sql ├── nft │ ├── ethereum │ │ ├── nft_ethereum_mints.sql │ │ ├── nft_ethereum_schema.yml │ │ └── nft_ethereum_transfers.sql │ ├── nft_mints.sql │ ├── nft_schema.yml │ └── nft_trades.sql ├── opensea │ ├── ethereum │ │ ├── opensea_ethereum_schema.yml │ │ ├── opensea_ethereum_sources.yml │ │ ├── opensea_ethereum_trades.sql │ │ └── opensea_ethereum_wyvern_atomic_match.sql │ ├── opensea_schema.yml │ └── opensea_trades.sql ├── prices │ └── prices_sources.yml └── tokens │ └── ethereum │ └── tokens_ethereum_sources.yml ├── packages.yml ├── profile └── profiles.yml ├── requirements_ci.txt ├── seeds └── nft │ ├── aggregators.csv │ └── schema.yml └── tests └── .gitkeep /.github/workflows/action-sqlfluff.yml: -------------------------------------------------------------------------------- 1 | name: sqlfluff with reviewdog 2 | 3 | on: 4 | pull_request: 5 | 6 | env: 7 | SPARK_DATABASE: ${{ secrets.SPARK_DATABASE }} 8 | SPARK_STS_HOST: ${{ secrets.SPARK_STS_HOST }} 9 | SPARK_STS_PORT: ${{ secrets.SPARK_STS_PORT }} 10 | 11 | jobs: 12 | test-check: 13 | name: runner / sqlfluff (github-check) 14 | runs-on: self-hosted 15 | steps: 16 | - uses: actions/checkout@v2 17 | - uses: yu-iskw/action-sqlfluff@v3.3.1 18 | id: lint-sql 19 | with: 20 | github_token: ${{ secrets.github_token }} 21 | reporter: github-pr-review 22 | sqlfluff_version: "0.12.0" 23 | sqlfluff_command: "fix" 24 | config: "${{ github.workspace }}/.sqlfluff-ci" 25 | paths: "${{ github.workspace }}/models" 26 | extra_requirements_txt: "${{ github.workspace }}/requirements_ci.txt" -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | dbt_modules/ 5 | logs/ 6 | profile/.user.yml 7 | .DS_Store -------------------------------------------------------------------------------- /.sqlfluff: -------------------------------------------------------------------------------- 1 | [sqlfluff] 2 | dialect = sparksql 3 | templater = dbt 4 | exclude_rules = L011, L014, L031 5 | 6 | [sqlfluff:templater:dbt] 7 | project_dir = ./ 8 | profiles_dir = ./profile 9 | target = dev 10 | 11 | [sqlfluff:rules] 12 | tab_space_size = 2 13 | max_line_length = 120 14 | comma_style = trailing 15 | 16 | [sqlfluff:rules:L010] 17 | # Keywords 18 | capitalisation_policy = lower 19 | 20 | [sqlfluff:rules:L047] 21 | # Count(1) 22 | prefer_count_1 = True 23 | 24 | [sqlfluff:rules:L030] 25 | # Function names 26 | capitalisation_policy = lower 27 | 28 | [sqlfluff:rules:L040] 29 | # Null & Boolean Literals 30 | capitalisation_policy = lower 31 | -------------------------------------------------------------------------------- /.sqlfluff-ci: -------------------------------------------------------------------------------- 1 | 2 | [sqlfluff:templater:dbt] 3 | profiles_dir = ./profile 4 | target = ci -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dbt_datawaves 2 | 3 | 4 | ## What does this dbt package do? 5 | 6 | 7 | This dbt package creates models on top of [Datawaves](https://datawaves.xyz/) [Decoded Projects Data](https://docs.datawaves.xyz/evm-blockchains/decoded-projects-data) that: 8 | 9 | * Translate smart contract function calls and events into domain models 10 | * Enrich trades and mints with details about associated the such as USD price for a token 11 | * Provide balance table at different time granularity 12 | 13 | 14 | ## Abstractions 15 | 16 | The pre-built abstractions data models can be divided into two categories. The project abstractions are project-level models built on top of the Decoded Projects Data in Datawaves. And the sector abstractions combine models from multiple projects into one table representing a domain, e.g., NFT. 17 | 18 | 19 | ### Project Models 20 | 21 | | Project | Models | description | Supported Chains | 22 | |---|---|---|---| 23 | | ens | [ens_registrations](https://github.com/datawaves-xyz/dbt_datawaves/blob/main/models/ens/ethereum/ens_ethereum_registrations.sql) | Each record represents a registration on ENS | Ethereum | 24 | | ens | [ens_reverse_registrars](https://github.com/datawaves-xyz/dbt_datawaves/blob/main/models/ens/ethereum/ens_ethereum_reverse_registrars.sql) | Each record represents a reverse registrar on ENS | Ethereum | 25 | | opensea | [opensea_trades](https://github.com/datawaves-xyz/dbt_datawaves/blob/main/models/opensea/opensea_trades.sql) | Each record represents a trade in OpenSeas, enriched with USD price of the trade. | Ethereum | 26 | | cryptopunks | [cryptopunks_trades](https://github.com/datawaves-xyz/dbt_datawaves/blob/main/models/cryptopunks/cryptopunks_trades.sql) | Each record represents a trade in CryptoPunksMarket, enriched with USD price of the trade. | Ethereum | 27 | 28 | 29 | ### Sector Models 30 | 31 | 32 | | Sector | Models | description | Supported Chains | 33 | |---|---|---|---| 34 | | nft | [nft_trades](https://github.com/datawaves-xyz/dbt_datawaves/blob/main/models/nft/nft_trades.sql) | Each record represents a trade, enriched with USD price of the trade. | Ethereum | 35 | | nft | [nft_mints](https://github.com/datawaves-xyz/dbt_datawaves/blob/main/models/nft/nft_mints.sql) | Each record represents an ERC721/ERC1155 token that has been minted. | Ethereum | 36 | | erc20 | [erc20_ethereum_transfers](https://github.com/datawaves-xyz/dbt_datawaves/blob/main/models/erc20/ethereum/erc20_ethereum_transfers.sql) | Each record represents an ERC20 token transfer event. | Ethereum | 37 | | erc721 | [erc721_ethereum_transfers](https://github.com/datawaves-xyz/dbt_datawaves/blob/main/models/erc721/ethereum/erc721_ethereum_transfers.sql) | Each record represents an ERC721 token transfer event. | Ethereum | 38 | | erc1155 | [erc1155_ethereum_transfers](https://github.com/datawaves-xyz/dbt_datawaves/blob/main/models/erc1155/ethereum/erc1155_ethereum_transfers.sql) | Each record represents an ERC1155 token transfer event. | Ethereum | 39 | 40 | 41 | ## Project using this package 42 | * [dbt_datawaves_wallet_labels](https://github.com/datawaves-xyz/dbt_datawaves_wallet_labels) 43 | 44 | 45 | ## How do I use this dbt package? 46 | 47 | ### Prerequisites 48 | 49 | To use this dbt project, you must have the following: 50 | 51 | * Use Datawaves ETL Cloud to sync data into your destination. 52 | * Make sure every source defined in `*_source.yml` exists in your destination. You can run `dbt source freshness` command to ensure they are "fresh". 53 | 54 | ![](./assets/dbt_datawaves_architecture.png) 55 | 56 | 57 | ### Install the package 58 | 59 | Include in your packages.yml: 60 | 61 | 62 | ```yml 63 | packages: 64 | - git: "https://github.com/datawaves-xyz/dbt_datawaves" 65 | revision: "0.0.1" 66 | ``` 67 | 68 | 69 | ## Contribute 70 | 71 | Additional contributions to this package are very welcome! Please create issues or open PRs against main. 72 | 73 | 74 | ## Database support 75 | 76 | 77 | This package has been tested on Databricks. 78 | 79 | -------------------------------------------------------------------------------- /assets/dbt_datawaves_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datawaves-xyz/dbt_datawaves/2738c66f862c9ae4551fd37ec97e4b255414ef96/assets/dbt_datawaves_architecture.png -------------------------------------------------------------------------------- /dbt_project.yml: -------------------------------------------------------------------------------- 1 | config-version: 2 2 | 3 | name: "datawaves" 4 | version: "1.0.0" 5 | profile: "datawaves" 6 | 7 | models: 8 | +file_format: delta 9 | datawaves: 10 | erc20: 11 | +schema: erc20 12 | +tags: ['sector'] 13 | ethereum: 14 | +schema: erc20_ethereum 15 | +materialized: view 16 | 17 | erc721: 18 | +schema: erc721 19 | +tags: ['sector'] 20 | ethereum: 21 | +schema: erc721_ethereum 22 | +materialized: view 23 | 24 | erc1155: 25 | +schema: erc1155 26 | +tags: ['sector'] 27 | ethereum: 28 | +schema: erc1155_ethereum 29 | +materialized: view 30 | 31 | nft: 32 | +schema: nft 33 | +materialized: view 34 | +tags: ['sector'] 35 | ethereum: 36 | +schema: nft_ethereum 37 | +materialized: view 38 | 39 | cryptopunks: 40 | +schema: cryptopunks 41 | +materialized: view 42 | +tags: ['project'] 43 | ethereum: 44 | +schema: cryptopunks_ethereum 45 | +materialized: view 46 | 47 | opensea: 48 | +schema: opensea 49 | +materialized: view 50 | +tags: ['project'] 51 | ethereum: 52 | +schema: opensea_ethereum 53 | +materialized: view 54 | ens: 55 | +schema: ens 56 | +materialized: view 57 | +tags: ['project'] 58 | ethereum: 59 | +schema: ens_ethereum 60 | +materialized: view 61 | 62 | seeds: 63 | datawaves: 64 | nft: 65 | +schema: nft 66 | -------------------------------------------------------------------------------- /integration_tests/dbt_project.yml: -------------------------------------------------------------------------------- 1 | config-version: 2 2 | 3 | name: 'ethereum_integration_tests' 4 | version: '1.0' 5 | profile: 'integration_tests' 6 | 7 | seeds: 8 | 9 | ethereum_integration_tests: 10 | utils: 11 | data_displayed_amount: 12 | data_hex: 13 | data_substring: 14 | 15 | 16 | -------------------------------------------------------------------------------- /integration_tests/macros/tests.sql: -------------------------------------------------------------------------------- 1 | {% test assert_equal(model, actual, expected) %} 2 | select * from {{ model }} where {{ actual }} != {{ expected }} 3 | 4 | {% endtest %} -------------------------------------------------------------------------------- /integration_tests/models/utils/schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: test_hex 5 | tests: 6 | - assert_equal: 7 | actual: actual 8 | expected: expected 9 | 10 | - name: test_displayed_amount 11 | tests: 12 | - assert_equal: 13 | actual: actual 14 | expected: expected 15 | 16 | - name: test_substring 17 | tests: 18 | - assert_equal: 19 | actual: actual 20 | expected: expected -------------------------------------------------------------------------------- /integration_tests/models/utils/test_binary_to_address.sql: -------------------------------------------------------------------------------- 1 | with data as ( 2 | select * from {{ ref("data_binary_to_address")}} 3 | ) 4 | 5 | select 6 | 7 | {{ ethereum.binary_to_address(ethereum.binary_literal("ffffffffffffffffffff")) }} as actual, 8 | address as expected 9 | from data -------------------------------------------------------------------------------- /integration_tests/models/utils/test_displayed_amount.sql: -------------------------------------------------------------------------------- 1 | with data as ( 2 | select * from {{ ref("data_displayed_amount")}} 3 | ) 4 | 5 | select 6 | 7 | {{ ethereum.displayed_amount("original_amount", "decimals") }} as actual, 8 | amount as expected 9 | from data -------------------------------------------------------------------------------- /integration_tests/models/utils/test_hex.sql: -------------------------------------------------------------------------------- 1 | with data as ( 2 | select * from {{ ref("data_hex")}} 3 | ) 4 | 5 | select 6 | 7 | {{ ethereum.hex(ethereum.binary_literal("01010101")) }} as actual, 8 | hex as expected 9 | from data -------------------------------------------------------------------------------- /integration_tests/models/utils/test_substring.sql: -------------------------------------------------------------------------------- 1 | with data as ( 2 | select * from {{ ref("data_substring")}} 3 | ) 4 | 5 | select 6 | 7 | {{ ethereum.substring("text", "position", "length") }} as actual, 8 | subtext as expected 9 | from data -------------------------------------------------------------------------------- /integration_tests/packages.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - local: ../ -------------------------------------------------------------------------------- /integration_tests/seeds/utils/data_binary_to_address.csv: -------------------------------------------------------------------------------- 1 | address 2 | "0xFFFFFFFFFFFFFFFFFFFF" -------------------------------------------------------------------------------- /integration_tests/seeds/utils/data_displayed_amount.csv: -------------------------------------------------------------------------------- 1 | original_amount,decimals,amount 2 | 15000000,7,1.5 -------------------------------------------------------------------------------- /integration_tests/seeds/utils/data_hex.csv: -------------------------------------------------------------------------------- 1 | hex 2 | "01010101" -------------------------------------------------------------------------------- /integration_tests/seeds/utils/data_substring.csv: -------------------------------------------------------------------------------- 1 | text,position,length,subtext 2 | "solidty",1,2,"ol" -------------------------------------------------------------------------------- /macros/generate_schema_name.sql: -------------------------------------------------------------------------------- 1 | {% macro generate_schema_name(custom_schema_name, node) -%} 2 | 3 | {%- set default_schema = target.schema -%} 4 | {%- if custom_schema_name is none -%} 5 | 6 | {{ default_schema }} 7 | 8 | {%- else -%} 9 | 10 | {{ custom_schema_name | trim }} 11 | 12 | {%- endif -%} 13 | 14 | {%- endmacro %} -------------------------------------------------------------------------------- /macros/utils/cte_import.sql: -------------------------------------------------------------------------------- 1 | {% macro cte_import(tuple_list) -%} 2 | 3 | with{% for cte_ref in tuple_list %} {{cte_ref[0]}} as ( 4 | select * 5 | from {{ ref(cte_ref[1]) }} 6 | ) 7 | {%- if not loop.last -%} 8 | , 9 | {%- endif -%} 10 | 11 | {%- endfor -%} 12 | 13 | {%- endmacro %} 14 | -------------------------------------------------------------------------------- /models/base/base_ethereum_sources.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | sources: 3 | - name: ethereum 4 | schema: ethereum_raw 5 | 6 | tables: 7 | - name: transactions 8 | freshness: 9 | error_after: {count: 48, period: hour} 10 | loaded_at_field: block_timestamp 11 | - name: blocks 12 | freshness: 13 | error_after: {count: 48, period: hour} 14 | loaded_at_field: timestamp 15 | - name: contracts 16 | - name: prices 17 | -------------------------------------------------------------------------------- /models/cryptopunks/cryptopunk_schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: cryptopunks_trades 5 | -------------------------------------------------------------------------------- /models/cryptopunks/cryptopunks_trades.sql: -------------------------------------------------------------------------------- 1 | {{config(alias='trades')}} 2 | 3 | select 4 | blockchain, 5 | platform, 6 | nft_token_id, 7 | exchange_contract_address, 8 | nft_contract_address, 9 | erc_standard, 10 | aggregator, 11 | number_of_items, 12 | trade_type, 13 | buyer, 14 | seller, 15 | nft_project_name, 16 | currency_amount, 17 | usd_amount, 18 | eth_amount, 19 | original_currency_amount, 20 | currency_symbol, 21 | currency_contract, 22 | original_currency_contract, 23 | block_time, 24 | block_number, 25 | tx_hash, 26 | tx_from, 27 | tx_to 28 | from {{ ref('cryptopunks_ethereum_trades') }} -------------------------------------------------------------------------------- /models/cryptopunks/ethereum/cryptopunk_ethereum_schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: cryptopunks_ethereum_trades 5 | - name: cryptopunks_ethereum_transfers -------------------------------------------------------------------------------- /models/cryptopunks/ethereum/cryptopunks_ethereum_sources.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | sources: 3 | - name: ethereum_cryptopunks 4 | schema: ethereum_cryptopunks 5 | 6 | tables: 7 | - name: crypto_punks_market_evt_assign 8 | - name: crypto_punks_market_evt_punk_bought 9 | - name: crypto_punks_market_evt_punk_bid_entered 10 | - name: crypto_punks_market_evt_punk_transfer 11 | -------------------------------------------------------------------------------- /models/cryptopunks/ethereum/cryptopunks_ethereum_trades.sql: -------------------------------------------------------------------------------- 1 | {{config(alias='trades')}} 2 | 3 | with tx as ( 4 | select * 5 | from {{ source('ethereum', 'transactions') }} 6 | ), 7 | 8 | punk_bought as ( 9 | select * 10 | from {{ source('ethereum_cryptopunks', 'crypto_punks_market_evt_punk_bought')}} 11 | ), 12 | 13 | punk_bid_entered as ( 14 | select * 15 | from {{ source('ethereum_cryptopunks', 'crypto_punks_market_evt_punk_bid_entered') }} 16 | ), 17 | 18 | erc20_token_transfers as ( 19 | select * 20 | from {{ source('ethereum_common', 'erc_20_evt_transfer') }} 21 | ), 22 | 23 | prices_usd as ( 24 | select * 25 | from {{ source('ethereum', 'prices') }} 26 | ), 27 | 28 | agg as ( 29 | select * 30 | from {{ ref('aggregators')}} 31 | ), 32 | 33 | address_info as ( 34 | select 35 | a.evt_block_time as block_time, 36 | a.evt_block_number as block_number, 37 | a.evt_tx_hash as tx_hash, 38 | a.punk_index as nft_token_id, 39 | a.from_address as from_address, 40 | cast(a.value as double) as value, 41 | case 42 | when a.to_address = '0x0000000000000000000000000000000000000000' then b.to 43 | else a.to_address 44 | end as to_address 45 | from punk_bought a 46 | left join erc20_token_transfers b 47 | on a.evt_tx_hash = b.evt_tx_hash and a.from_address = b.from 48 | and b.contract_address = '0xb47e3cd837ddf8e4c57f05d70ab865de6e193bbb' 49 | ), 50 | 51 | punk_trade as ( 52 | select 53 | block_time, 54 | block_number, 55 | tx_hash, 56 | nft_token_id, 57 | from_address, 58 | to_address, 59 | (case when value = 0 then coalesce(bid_value, 0) else value end) as original_currency_amount 60 | from ( 61 | select 62 | a.*, 63 | cast(b.value as double) as bid_value, 64 | row_number()over(partition by a.nft_token_id, a.block_time order by b.evt_block_time desc) as rank 65 | from address_info a 66 | left join punk_bid_entered b 67 | on a.nft_token_id = b.punk_index and b.from_address = a.to_address and b.evt_block_time <= a.block_time 68 | ) 69 | where rank = 1 70 | ), 71 | 72 | punk_agg_tx as ( 73 | select 74 | tx_hash, 75 | block_time, 76 | block_number, 77 | count(distinct nft_token_id) as num 78 | from punk_trade 79 | group by tx_hash, block_time, block_number 80 | ) 81 | 82 | select 83 | 'Ethereum' as blockchain, 84 | 'LarvaLabs Contract' as platform, 85 | a.nft_token_id, 86 | '0xb47e3cd837ddf8e4c57f05d70ab865de6e193bbb' as exchange_contract_address, 87 | '0xb47e3cd837ddf8e4c57f05d70ab865de6e193bbb' as nft_contract_address, 88 | 'erc20' as erc_standard, 89 | b.num as number_of_items, 90 | agg.name as aggregator, 91 | case when b.num > 1 then 'Bundle Trade' else 'Single Item Trade' end as trade_type, 92 | a.from_address as buyer, 93 | a.to_address as seller, 94 | 'CryptoPunks' as nft_project_name, 95 | a.original_currency_amount / power(10, 18) as currency_amount, 96 | 'ETH' as currency_symbol, 97 | '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' as currency_contract, 98 | a.original_currency_amount / power(10, 18) * p.price as usd_amount, 99 | a.original_currency_amount / power(10, 18) as eth_amount, 100 | a.original_currency_amount, 101 | '0x0000000000000000000000000000000000000000' as original_currency_contract, 102 | a.block_time, 103 | a.block_number, 104 | a.tx_hash, 105 | tx.from_address as tx_from, 106 | tx.to_address as tx_to 107 | from punk_trade a 108 | left join punk_agg_tx b 109 | on a.tx_hash = b.tx_hash 110 | left join tx 111 | on a.tx_hash = tx.hash 112 | left join prices_usd p 113 | on p.minute = {{ dbt_utils.date_trunc('minute', 'a.block_time') }} 114 | and p.contract_address = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' 115 | left join agg 116 | on agg.contract_address = tx.to_address 117 | -------------------------------------------------------------------------------- /models/cryptopunks/ethereum/cryptopunks_ethereum_transfers.sql: -------------------------------------------------------------------------------- 1 | {{config(alias='transfers')}} 2 | 3 | select 4 | contract_address as nft_contract_address, 5 | punk_index as nft_token_id, 6 | to as to_address, 7 | evt_block_time as block_time 8 | from {{ source('ethereum_cryptopunks', 'crypto_punks_market_evt_punk_transfer') }} 9 | 10 | union distinct 11 | 12 | select 13 | contract_address as nft_contract_address, 14 | punk_index as nft_token_id, 15 | to_address, 16 | evt_block_time as block_time 17 | from {{ source('ethereum_cryptopunks', 'crypto_punks_market_evt_punk_bought') }} 18 | 19 | union distinct 20 | 21 | select 22 | contract_address as nft_contract_address, 23 | punk_index as nft_token_id, 24 | to as to_address, 25 | evt_block_time as block_time 26 | from {{ source('ethereum_cryptopunks', 'crypto_punks_market_evt_assign') }} -------------------------------------------------------------------------------- /models/ens/ens_registrations.sql: -------------------------------------------------------------------------------- 1 | {{config(alias='registrations')}} 2 | 3 | select 4 | * 5 | from {{ ref('ens_ethereum_registrations') }} -------------------------------------------------------------------------------- /models/ens/ens_reverse_registrars.sql: -------------------------------------------------------------------------------- 1 | {{config(alias='reverse_registrars')}} 2 | 3 | select 4 | * 5 | from {{ ref('ens_ethereum_reverse_registrars') }} -------------------------------------------------------------------------------- /models/ens/ens_schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: ens_registrations 5 | - name: ens_reverse_registrars 6 | -------------------------------------------------------------------------------- /models/ens/ethereum/ens_ethereum_registrations.sql: -------------------------------------------------------------------------------- 1 | {{config(alias='registrations')}} 2 | 3 | select * 4 | from {{source('ethereum_ens', 'ethregistrar_controller_evt_name_registered')}} 5 | union 6 | select * 7 | from {{source('ethereum_ens', 'ethregistrar_controller_2_evt_name_registered')}} 8 | union 9 | select * 10 | from {{source('ethereum_ens', 'ethregistrar_controller_3_evt_name_registered')}} -------------------------------------------------------------------------------- /models/ens/ethereum/ens_ethereum_reverse_registrars.sql: -------------------------------------------------------------------------------- 1 | with txn as ( 2 | select 3 | * from {{ source('ethereum', 'transactions') }} 4 | ), 5 | 6 | ens_reverse_registrar as ( 7 | select 8 | * from {{ source('ethereum_ens', 'reverse_registrar_1_call_set_name') }} 9 | union 10 | select 11 | * from {{ source('ethereum_ens', 'reverse_registrar_2_call_set_name') }} 12 | ), 13 | 14 | ens_txn as ( 15 | select 16 | distinct from_address as eth_addr, 17 | block_number, 18 | block_timestamp, 19 | hash as tx_hash 20 | from txn 21 | where 22 | -- Old Reverse Registrar and Reverse Registrar contracts 23 | (to_address = '0x9062c0a6dbd6108336bcbe4593a3d1ce05512069' or to_address = '0x084b1c3c81545d370f3634392de611caabff8148') 24 | -- Only successful transactions 25 | and receipt_status = 1 26 | ), 27 | 28 | {{config(alias='reverse_registrars')}} 29 | 30 | all_reverse_registrar as ( 31 | select 32 | distinct name as ens_name, 33 | call_block_number as block_number, 34 | call_tx_hash as tx_hash 35 | from ens_reverse_registrar 36 | -- Only successful transactions 37 | where call_success is True 38 | ) 39 | 40 | select 41 | t.eth_addr as address, 42 | t.block_timestamp, 43 | t.tx_hash, 44 | a.ens_name 45 | from ens_txn as t 46 | inner join all_reverse_registrar as a 47 | on a.block_number = t.block_number 48 | and a.tx_hash = t.tx_hash 49 | and a.ens_name <> '0x0000000000000000000000000000000000000000' 50 | -------------------------------------------------------------------------------- /models/ens/ethereum/ens_ethereum_schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | models: 3 | - name: ens_ethereum_registrations 4 | - name: ens_ethereum_reverse_registrars -------------------------------------------------------------------------------- /models/ens/ethereum/ens_ethereum_sources.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | sources: 3 | - name: ethereum_ens 4 | schema: ethereum_ens 5 | tables: 6 | - name: ethregistrar_controller_evt_name_registered 7 | - name: ethregistrar_controller_2_evt_name_registered 8 | - name: ethregistrar_controller_3_evt_name_registered 9 | - name: reverse_registrar_1_call_set_name 10 | - name: reverse_registrar_2_call_set_name -------------------------------------------------------------------------------- /models/erc1155/ethereum/erc1155_ethereum_schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: erc1155_ethereum_transfers -------------------------------------------------------------------------------- /models/erc1155/ethereum/erc1155_ethereum_sources.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | sources: 3 | - name: ethereum_common 4 | schema: ethereum_common 5 | freshness: 6 | error_after: {count: 48, period: hour} 7 | loaded_at_field: evt_block_time 8 | tables: 9 | - name: erc_1155_evt_transfer_single 10 | - name: erc_1155_evt_transfer_batch -------------------------------------------------------------------------------- /models/erc1155/ethereum/erc1155_ethereum_transfers.sql: -------------------------------------------------------------------------------- 1 | {{config(alias='transfers')}} 2 | 3 | with erc1155_transfers_batch as ( 4 | select 5 | distinct erc1155_ids_batch.exploded_id, 6 | erc1155_values_batch.exploded_value, 7 | erc1155_ids_batch.to, 8 | erc1155_ids_batch.from, 9 | erc1155_ids_batch.contract_address, 10 | erc1155_ids_batch.evt_block_time, 11 | erc1155_ids_batch.evt_tx_hash 12 | from (select *, explode(ids) as exploded_id 13 | from {{ source('ethereum_common', 'erc_1155_evt_transfer_batch') }}) erc1155_ids_batch 14 | join (select *, explode(values) as exploded_value 15 | from {{ source('ethereum_common', 'erc_1155_evt_transfer_batch') }}) erc1155_values_batch 16 | on erc1155_ids_batch.evt_tx_hash = erc1155_values_batch.evt_tx_hash 17 | and erc1155_ids_batch.evt_index = erc1155_values_batch.evt_index 18 | ), 19 | 20 | erc1155_transfers as ( 21 | select 22 | from, 23 | to, 24 | cast(id as {{ dbt_utils.type_string() }}) as token_id, 25 | cast(value as double) as amount, 26 | contract_address as token_address, 27 | evt_block_time as block_time, 28 | evt_tx_hash as tx_hash 29 | from {{ source('ethereum_common', 'erc_1155_evt_transfer_single') }} 30 | 31 | union all 32 | 33 | select 34 | from, 35 | to, 36 | cast(exploded_id as {{ dbt_utils.type_string() }}) as token_id, 37 | cast(exploded_value as double) as amount, 38 | contract_address as token_address, 39 | evt_block_time as block_time, 40 | evt_tx_hash as tx_hash 41 | from erc1155_transfers_batch 42 | ) 43 | 44 | select * 45 | from erc1155_transfers 46 | -------------------------------------------------------------------------------- /models/erc20/ethereum/erc20_ethereum_schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: erc20_ethereum_transfers -------------------------------------------------------------------------------- /models/erc20/ethereum/erc20_ethereum_sources.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | sources: 3 | - name: ethereum_common 4 | schema: ethereum_common 5 | freshness: 6 | error_after: {count: 48, period: hour} 7 | loaded_at_field: evt_block_time 8 | tables: 9 | - name: erc_20_evt_transfer -------------------------------------------------------------------------------- /models/erc20/ethereum/erc20_ethereum_transfers.sql: -------------------------------------------------------------------------------- 1 | {{config(alias='transfers')}} 2 | 3 | with erc20_transfers as ( 4 | select 5 | from, 6 | to, 7 | cast(value as double) as amount, 8 | contract_address as token_address, 9 | evt_block_time as block_time, 10 | evt_tx_hash as tx_hash 11 | from {{ source('ethereum_common', 'erc_20_evt_transfer') }} 12 | ) 13 | 14 | select * 15 | from erc20_transfers -------------------------------------------------------------------------------- /models/erc721/ethereum/erc721_ethereum_schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: erc721_ethereum_transfers -------------------------------------------------------------------------------- /models/erc721/ethereum/erc721_ethereum_sources.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | sources: 3 | - name: ethereum_common 4 | schema: ethereum_common 5 | freshness: 6 | error_after: {count: 48, period: hour} 7 | loaded_at_field: evt_block_time 8 | tables: 9 | - name: erc_721_evt_transfer 10 | -------------------------------------------------------------------------------- /models/erc721/ethereum/erc721_ethereum_transfers.sql: -------------------------------------------------------------------------------- 1 | {{config(alias='transfers')}} 2 | 3 | with erc721_transfers as ( 4 | select 5 | from, 6 | to, 7 | cast(token_id as {{ dbt_utils.type_string() }}) as token_id, 8 | cast(1 as double) as amount, 9 | contract_address as token_address, 10 | evt_block_time as block_time, 11 | evt_tx_hash as tx_hash 12 | from {{ source('ethereum_common', 'erc_721_evt_transfer') }} 13 | ) 14 | 15 | select * 16 | from erc721_transfers -------------------------------------------------------------------------------- /models/nft/ethereum/nft_ethereum_mints.sql: -------------------------------------------------------------------------------- 1 | {{config(alias='mints')}} 2 | 3 | with transactions as ( 4 | select * 5 | from {{ source('ethereum', 'transactions') }} 6 | ), 7 | 8 | prices_usd as ( 9 | select * 10 | from {{ source('ethereum', 'prices') }} 11 | ), 12 | 13 | mint_tx as ( 14 | select 15 | a.hash as tx_hash, 16 | a.value as tx_amount, 17 | b.token_address as nft_contract_address, 18 | b.token_id as nft_token_id, 19 | b.block_time, 20 | b.to as minter, 21 | b.amount as quantity, 22 | b.erc_standard 23 | from transactions as a 24 | join {{ ref('nft_ethereum_transfers') }} as b 25 | on a.hash = b.tx_hash and b.`type` = 'Mint' 26 | ), 27 | 28 | mint as ( 29 | select 30 | x.erc_standard, 31 | x.tx_hash, 32 | x.nft_contract_address, 33 | x.nft_token_id, 34 | x.quantity, 35 | x.block_time, 36 | x.minter, 37 | x.tx_amount, 38 | num_of_items, 39 | ({{ datawaves_utils.displayed_amount('x.tx_amount', 18) }} / num_of_items) as eth_mint_price 40 | from mint_tx as x 41 | left join ( 42 | select 43 | tx_hash, 44 | count(distinct nft_token_id) as num_of_items 45 | from mint_tx 46 | group by tx_hash 47 | ) as y 48 | on x.tx_hash = y.tx_hash 49 | ) 50 | 51 | select 52 | 'Ethereum' as blockchain, 53 | u.erc_standard, 54 | u.tx_hash, 55 | u.nft_contract_address, 56 | u.nft_token_id, 57 | u.quantity, 58 | u.block_time, 59 | u.minter, 60 | u.tx_amount, 61 | u.num_of_items, 62 | u.eth_mint_price, 63 | u.eth_mint_price * p.price as usd_mint_price 64 | from mint as u 65 | left join prices_usd as p 66 | on p.minute = {{ dbt_utils.date_trunc('minute', 'u.block_time') }} 67 | and p.contract_address = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' -------------------------------------------------------------------------------- /models/nft/ethereum/nft_ethereum_schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: nft_ethereum_mints 5 | - name: nft_ethereum_transfers -------------------------------------------------------------------------------- /models/nft/ethereum/nft_ethereum_transfers.sql: -------------------------------------------------------------------------------- 1 | {{config(alias='transfers')}} 2 | 3 | with nft_transfers as ( 4 | select 'erc1155' as erc_standard, * 5 | from {{ ref('erc1155_ethereum_transfers') }} 6 | 7 | union all 8 | 9 | select 'erc721' as erc_standard, * 10 | from {{ ref('erc721_ethereum_transfers') }} 11 | ) 12 | 13 | select 14 | 'Ethereum' as blockchain, 15 | case 16 | when `from` = '0x0000000000000000000000000000000000000000' then 'Mint' 17 | when `to` = '0x0000000000000000000000000000000000000000' 18 | or `to` = '0x000000000000000000000000000000000000dead' then 'Burn' 19 | else 'Trade' 20 | end as `type`, 21 | * 22 | from nft_transfers -------------------------------------------------------------------------------- /models/nft/nft_mints.sql: -------------------------------------------------------------------------------- 1 | {{config(alias='mints')}} 2 | 3 | select * 4 | from {{ ref("nft_ethereum_mints") }} -------------------------------------------------------------------------------- /models/nft/nft_schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: nft_trades 5 | - name: nft_mints 6 | -------------------------------------------------------------------------------- /models/nft/nft_trades.sql: -------------------------------------------------------------------------------- 1 | {{config(alias='trades')}} 2 | 3 | select * 4 | from {{ ref('opensea_trades') }} 5 | 6 | union all 7 | 8 | select * 9 | from {{ ref('cryptopunks_trades') }} -------------------------------------------------------------------------------- /models/opensea/ethereum/opensea_ethereum_schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: opensea_ethereum_wyvern_atomic_match 5 | - name: opensea_ethereum_trades 6 | -------------------------------------------------------------------------------- /models/opensea/ethereum/opensea_ethereum_sources.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | sources: 3 | - name: ethereum_opensea 4 | schema: ethereum_opensea 5 | freshness: 6 | error_after: {count: 48, period: hour} 7 | loaded_at_field: call_block_time 8 | tables: 9 | - name: wyvern_exchange_call_atomic_match_ 10 | - name: wyvern_exchange_2_call_atomic_match_ -------------------------------------------------------------------------------- /models/opensea/ethereum/opensea_ethereum_trades.sql: -------------------------------------------------------------------------------- 1 | {{config(alias='trades')}} 2 | 3 | with nft_tokens as ( 4 | select * 5 | from {{ source('ethereum_tokens', 'nft') }} 6 | ), 7 | 8 | erc20 as ( 9 | select * 10 | from {{ source('ethereum_tokens', 'erc20') }} 11 | ), 12 | 13 | erc721_token_transfers as ( 14 | select * 15 | from {{ source('ethereum_common', 'erc_721_evt_transfer') }} 16 | ), 17 | 18 | erc1155_token_transfers as ( 19 | select * 20 | from {{ source('ethereum_common', 'erc_1155_evt_transfer_single') }} 21 | ), 22 | 23 | prices_usd as ( 24 | select * 25 | from {{ source('ethereum', 'prices') }} 26 | ), 27 | 28 | wyvern_data as ( 29 | select * 30 | from {{ ref('opensea_ethereum_wyvern_atomic_match') }} 31 | ), 32 | 33 | agg as ( 34 | select * 35 | from {{ ref('aggregators')}} 36 | ), 37 | 38 | -- Count number of token IDs in each transaction 39 | erc721_tokens_in_tx as ( 40 | select 41 | evt_tx_hash as tx_hash, 42 | token_id as token_id, 43 | count(1) as num_of_items 44 | from erc721_token_transfers 45 | where `from` != '0x0000000000000000000000000000000000000000' 46 | group by evt_tx_hash, token_id 47 | ), 48 | 49 | -- Count number of token IDs in each transaction 50 | erc1155_tokens_in_tx as ( 51 | select 52 | evt_tx_hash as tx_hash, 53 | id as token_id, 54 | count(1) as num_of_items 55 | from erc1155_token_transfers 56 | where `from` != '0x0000000000000000000000000000000000000000' 57 | group by evt_tx_hash, id 58 | ) 59 | 60 | select 61 | 'Ethereum' as blockchain, 62 | 'OpenSea' as platform, 63 | w.token_id as nft_token_id, 64 | w.exchange_contract_address, 65 | w.nft_contract_address, 66 | case 67 | when erc1155_tokens_in_tx.num_of_items >= 1 then 'erc1155' 68 | when erc721_tokens_in_tx.num_of_items >= 1 then 'erc721' 69 | else w.erc_standard 70 | end as erc_standard, 71 | agg.name as aggregator, 72 | -- Count the number of items for different trade types 73 | case 74 | when agg.name is null and erc721_tokens_in_tx.num_of_items > 1 then erc721_tokens_in_tx.num_of_items 75 | when agg.name is null and erc1155_tokens_in_tx.num_of_items > 1 then erc1155_tokens_in_tx.num_of_items 76 | when w.trade_type = 'Single Item Trade' then 1 77 | when w.erc_standard = 'erc1155' then erc721_tokens_in_tx.num_of_items 78 | when w.erc_standard = 'erc721' then erc1155_tokens_in_tx.num_of_items 79 | else 80 | (select count(1) 81 | from erc721_token_transfers 82 | where evt_tx_hash = w.tx_hash) 83 | + 84 | (select count(1) 85 | from erc1155_token_transfers 86 | where evt_tx_hash = w.tx_hash) 87 | end as number_of_items, 88 | -- A bundle trade contains at least one erc721 or erc1155 tokens in tx 89 | case 90 | when agg.name is not null then 'Aggregator Trade' 91 | when erc721_tokens_in_tx.num_of_items = 1 or erc1155_tokens_in_tx.num_of_items = 1 then 'Single Item Trade' 92 | when erc721_tokens_in_tx.num_of_items > 1 or erc1155_tokens_in_tx.num_of_items > 1 then 'Bundle Trade' 93 | else w.trade_type 94 | end as trade_type, 95 | -- Replace the buyer when using aggregator to trade 96 | case when agg.name is not null then w.buyer_when_aggr 97 | else w.buyer 98 | end as buyer, 99 | w.seller, 100 | -- Get the token of aggregator when using aggregator to trade 101 | case 102 | when agg.name is not null then agg_tokens.symbol 103 | else nft_tokens.name 104 | end as nft_project_name, 105 | -- Adjust the currency amount/symbol with erc20 tokens 106 | {{ datawaves_utils.displayed_amount('w.currency_amount', 'erc20.decimals') }} as currency_amount, 107 | {{ datawaves_utils.displayed_amount('w.currency_amount', 'erc20.decimals') }} * p.price as usd_amount, 108 | {{ datawaves_utils.displayed_amount('w.currency_amount', 'erc20.decimals') }} * p.price / pe.price as eth_amount, 109 | w.currency_amount as original_currency_amount, 110 | case 111 | when w.original_currency_contract = '0x0000000000000000000000000000000000000000' 112 | then 'ETH' 113 | else erc20.symbol 114 | end as currency_symbol, 115 | w.currency_contract, 116 | w.original_currency_contract, 117 | -- blocks & tx 118 | w.block_time, 119 | w.block_number, 120 | w.tx_hash, 121 | w.tx_from, 122 | w.tx_to 123 | from wyvern_data w 124 | 125 | left join erc721_tokens_in_tx 126 | on erc721_tokens_in_tx.tx_hash = w.tx_hash 127 | and erc721_tokens_in_tx.token_id = w.token_id 128 | 129 | left join erc1155_tokens_in_tx 130 | on erc1155_tokens_in_tx.tx_hash = w.tx_hash 131 | and erc1155_tokens_in_tx.token_id = w.token_id 132 | 133 | left join prices_usd p 134 | on p.minute = {{ dbt_utils.date_trunc('minute', 'w.block_time') }} 135 | and p.contract_address = w.currency_contract 136 | 137 | left join prices_usd pe 138 | on pe.minute = {{ dbt_utils.date_trunc('minute', 'w.block_time') }} 139 | and pe.symbol = 'WETH' 140 | 141 | left join erc20 on erc20.contract_address = w.currency_contract 142 | left join nft_tokens on nft_tokens.contract_address = w.nft_contract_address 143 | left join erc20 agg_tokens on agg_tokens.contract_address = w.nft_contract_address 144 | left join agg on agg.contract_address = w.tx_to 145 | -------------------------------------------------------------------------------- /models/opensea/ethereum/opensea_ethereum_wyvern_atomic_match.sql: -------------------------------------------------------------------------------- 1 | with wyvern_atomic_match as ( 2 | select * 3 | from {{ source('ethereum_opensea', 'wyvern_exchange_call_atomic_match_') }} 4 | union 5 | select * 6 | from {{ source('ethereum_opensea', 'wyvern_exchange_2_call_atomic_match_') }} 7 | ), 8 | 9 | tx as ( 10 | select * 11 | from {{ source('ethereum', 'transactions') }} 12 | ), 13 | 14 | wyvern_data as ( 15 | select 16 | call_tx_hash as tx_hash, 17 | call_block_number as block_number, 18 | call_block_time as block_time, 19 | addrs[1] as buyer, 20 | {{ datawaves_utils.binary_to_address(datawaves_utils.substring('calldata_buy', 49, 20)) }} as buyer_when_aggr, 21 | addrs[8] as seller, 22 | cast(uints[4] as double) as currency_amount, 23 | case 24 | when {{ datawaves_utils.substring('calldata_buy', 1, 4) }} in ({{ datawaves_utils.binary_literal('68f0bcaa') }}) then 'Bundle Trade' 25 | else 'Single Item Trade' 26 | end as trade_type, 27 | case 28 | when {{ datawaves_utils.substring('calldata_buy', 1, 4) }} in ({{ datawaves_utils.binary_literal('fb16a595') }}, {{ datawaves_utils.binary_literal('23b872dd') }}) 29 | then 'erc721' 30 | when {{ datawaves_utils.substring('calldata_buy', 1, 4) }} in ({{ datawaves_utils.binary_literal('23b872dd') }}, {{ datawaves_utils.binary_literal('f242432a') }}) 31 | then 'erc1155' 32 | end as erc_standard, 33 | addrs[0] as exchange_contract_address, 34 | case 35 | when {{ datawaves_utils.substring('calldata_buy', 1, 4) }} in ({{ datawaves_utils.binary_literal('fb16a595') }}, {{ datawaves_utils.binary_literal('96809f90') }}) 36 | then {{ datawaves_utils.binary_to_address(datawaves_utils.substring('calldata_buy', 81, 20)) }} 37 | else addrs[4] 38 | end as nft_contract_address, 39 | case 40 | when addrs[6] = '0x0000000000000000000000000000000000000000' then '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' 41 | else addrs[6] 42 | end as currency_contract, 43 | addrs[6] as original_currency_contract, 44 | case 45 | when {{ datawaves_utils.substring('calldata_buy', 1, 4) }} in ({{ datawaves_utils.binary_literal('fb16a595') }}, {{ datawaves_utils.binary_literal('96809f90') }}) 46 | then cast(round({{ datawaves_utils.binary_to_numeric(datawaves_utils.substring('calldata_buy', 101, 32)) }}, 0) as {{ dbt_utils.type_string() }}) 47 | when {{ datawaves_utils.substring('calldata_buy', 1, 4) }} in ({{ datawaves_utils.binary_literal('23b872dd') }}, {{ datawaves_utils.binary_literal('f242432a') }}) 48 | then cast(round({{ datawaves_utils.binary_to_numeric(datawaves_utils.substring('calldata_buy', 69, 32)) }}, 0) as {{ dbt_utils.type_string() }}) 49 | end as token_id 50 | 51 | from wyvern_atomic_match 52 | where 53 | (addrs[3] = '0x5b3256965e7c3cf26e11fcaf296dfc8807c01073' 54 | or addrs[10] = '0x5b3256965e7c3cf26e11fcaf296dfc8807c01073') 55 | and call_success = true 56 | ) 57 | 58 | select 59 | w.tx_hash, 60 | w.block_number, 61 | w.block_time, 62 | w.buyer, 63 | w.buyer_when_aggr, 64 | w.seller, 65 | w.currency_amount, 66 | w.trade_type, 67 | w.erc_standard, 68 | w.exchange_contract_address, 69 | w.nft_contract_address, 70 | w.currency_contract, 71 | w.original_currency_contract, 72 | w.token_id, 73 | tx.from_address as tx_from, 74 | tx.to_address as tx_to 75 | from wyvern_data w 76 | 77 | left join tx on tx.hash = w.tx_hash -------------------------------------------------------------------------------- /models/opensea/opensea_schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: opensea_trades 5 | -------------------------------------------------------------------------------- /models/opensea/opensea_trades.sql: -------------------------------------------------------------------------------- 1 | {{config(alias='trades')}} 2 | 3 | select 4 | blockchain, 5 | platform, 6 | nft_token_id, 7 | exchange_contract_address, 8 | nft_contract_address, 9 | erc_standard, 10 | aggregator, 11 | number_of_items, 12 | trade_type, 13 | buyer, 14 | seller, 15 | nft_project_name, 16 | currency_amount, 17 | usd_amount, 18 | eth_amount, 19 | original_currency_amount, 20 | currency_symbol, 21 | currency_contract, 22 | original_currency_contract, 23 | block_time, 24 | block_number, 25 | tx_hash, 26 | tx_from, 27 | tx_to 28 | from {{ ref('opensea_ethereum_trades') }} -------------------------------------------------------------------------------- /models/prices/prices_sources.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | sources: 3 | - name: prices 4 | schema: prices 5 | tables: 6 | - name: usd -------------------------------------------------------------------------------- /models/tokens/ethereum/tokens_ethereum_sources.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | sources: 3 | - name: ethereum_tokens 4 | schema: ethereum_tokens 5 | tables: 6 | - name: erc20 7 | - name: nft 8 | -------------------------------------------------------------------------------- /packages.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - git: "https://github.com/datawaves-xyz/dbt_datawaves_utils" 3 | -------------------------------------------------------------------------------- /profile/profiles.yml: -------------------------------------------------------------------------------- 1 | dbt_ethereum: 2 | target: prod 3 | outputs: 4 | prod: 5 | type: spark 6 | method: thrift 7 | threads: 4 8 | host: "{{ env_var('SPARK_STS_HOST', 'not-set') }}" 9 | port: "{{ env_var('SPARK_STS_PORT', '-1') | as_number }}" 10 | # dbt understands spark database as schema 11 | schema: "{{ env_var('SPARK_DATABASE', 'not-set') }}" 12 | ci: 13 | type: spark 14 | method: thrift 15 | threads: 4 16 | host: "{{ env_var('SPARK_STS_HOST', 'not-set') }}" 17 | port: "{{ env_var('SPARK_STS_PORT', '-1') | as_number }}" 18 | # dbt understands spark database as schema 19 | schema: "{{ env_var('SPARK_DATABASE', 'not-set') }}" 20 | dev: 21 | type: spark 22 | method: thrift 23 | threads: 4 24 | host: "{{ env_var('SPARK_STS_HOST', 'not-set') }}" 25 | port: "{{ env_var('SPARK_STS_PORT', '-1') | as_number }}" 26 | schema: "mock" -------------------------------------------------------------------------------- /requirements_ci.txt: -------------------------------------------------------------------------------- 1 | dbt-databricks -------------------------------------------------------------------------------- /seeds/nft/aggregators.csv: -------------------------------------------------------------------------------- 1 | contract_address,name 2 | "0x0a267cf51ef038fc00e71801f5a524aec06e4f07","GenieSwap" 3 | "0x0000000031f7382a812c64b604da4fc520afef4b","Gem" 4 | "0xf24629fbb477e10f2cf331c2b7452d8596b5c7a5","Gem" 5 | "0x83c8f28c26bf6aaca652df1dbbe0e1b56f8baba2","Gem" 6 | -------------------------------------------------------------------------------- /seeds/nft/schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | seeds: 4 | - name: aggregators 5 | -------------------------------------------------------------------------------- /tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datawaves-xyz/dbt_datawaves/2738c66f862c9ae4551fd37ec97e4b255414ef96/tests/.gitkeep --------------------------------------------------------------------------------