├── blockchainetl ├── __init__.py ├── jobs │ ├── __init__.py │ ├── exporters │ │ ├── __init__.py │ │ ├── converters │ │ │ ├── __init__.py │ │ │ ├── unix_timestamp_item_converter.py │ │ │ ├── simple_item_converter.py │ │ │ ├── composite_item_converter.py │ │ │ ├── int_to_string_item_converter.py │ │ │ ├── int_to_decimal_item_converter.py │ │ │ └── list_field_item_converter.py │ │ ├── console_item_exporter.py │ │ ├── in_memory_item_exporter.py │ │ ├── kafka_exporter.py │ │ └── postgres_item_exporter.py │ └── base_job.py ├── logging_utils.py ├── atomic_counter.py ├── csv_utils.py └── file_utils.py ├── .github ├── CODEOWNERS └── workflows │ ├── pytest.yml │ └── publish.yml ├── tests ├── resources │ ├── test_export_traces_job │ │ └── block_without_transactions │ │ │ ├── expected_traces.json │ │ │ └── web3_response.debug_traceBlockByNumber_0x1_timeout_1h_tracer_fastcalltracer.json │ ├── test_export_blocks_job │ │ └── block_without_transactions │ │ │ └── expected_transactions.json │ ├── test_export_tokens_job │ │ ├── erc20_tokens │ │ │ ├── web3_response.eth_chainId_.json │ │ │ ├── web3_response.eth_call_data_0x18160ddd_to_0x9e481eb17d3c3c07d7a6ab571b4ba8ef432b5cf2_latest.json │ │ │ ├── web3_response.eth_call_data_0x313ce567_to_0x9e481eb17d3c3c07d7a6ab571b4ba8ef432b5cf2_latest.json │ │ │ ├── expected_tokens.json │ │ │ ├── web3_response.eth_call_data_0x06fdde03_to_0x9e481eb17d3c3c07d7a6ab571b4ba8ef432b5cf2_latest.json │ │ │ └── web3_response.eth_call_data_0x95d89b41_to_0x9e481eb17d3c3c07d7a6ab571b4ba8ef432b5cf2_latest.json │ │ ├── erc721_tokens │ │ │ ├── web3_response.eth_chainId_.json │ │ │ ├── expected_tokens.json │ │ │ ├── web3_response.eth_call_data_0x18160ddd_to_0xb88168dde0001be8546c70c117ab9e41e28f7164_latest.json │ │ │ ├── web3_response.eth_call_data_0x06fdde03_to_0xb88168dde0001be8546c70c117ab9e41e28f7164_latest.json │ │ │ └── web3_response.eth_call_data_0x95d89b41_to_0xb88168dde0001be8546c70c117ab9e41e28f7164_latest.json │ │ └── erc1155_tokens │ │ │ └── expected_tokens.json │ ├── test_export_token_transfers_job │ │ └── block_with_transfers │ │ │ ├── web3_response.eth_uninstallFilter_0x849659f71ba861ad7a3e6b45e34a5045.json │ │ │ ├── web3_response.eth_newFilter_fromBlock_0x4d67c2e_toBlock_0x4d67c2e_topics_0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef.json │ │ │ └── expected_token_transfers.csv │ ├── test_export_trace_groups_job │ │ ├── trace_groups_raw │ │ │ └── expected_tokens.json │ │ └── trace_groups_enrich │ │ │ └── expected_tokens.json │ ├── test_export_receipts_job │ │ └── receipts │ │ │ ├── web3_response.klay_getTransactionReceipt_0x19485bb473e7d90af7d3a67d6631f91f5abf6a6a1e24ff24a3dfe30c5028e168.json │ │ │ ├── web3_response.klay_getTransactionReceipt_0x16fe9780d7d159958425877870e0a9db3187392b2386b1e81463ae03b8d92512.json │ │ │ ├── web3_response.klay_getTransactionReceipt_0x19613c8cf5a711cc3701dc5346ad095171150601abfcaf8e74ecbd536d4e4834.json │ │ │ ├── web3_response.klay_getTransactionReceipt_0xa4d4c825506b9efaf59bc596c9547fc1a4587ef45bac0aed730ad718434130c6.json │ │ │ ├── web3_response.klay_getTransactionReceipt_0x9b23a9ea174552fa158b41fab89be404b6b29aa5f24c2217e94625cf8a1c9a54.json │ │ │ ├── web3_response.klay_getTransactionReceipt_0x090a5a01702402a09481eb7b3c5a895cbb67aa6a6bd2372c334cb8e7ce337571.json │ │ │ ├── web3_response.klay_getTransactionReceipt_0xd6960970e0d741b37e2832d03ed4f6a935862fe9cea584e116ec3964205a654f.json │ │ │ ├── web3_response.klay_getTransactionReceipt_0x0d0cc98bb9460a70cb649c221e869d186b5b06048be750bc5a997c96d6d2071d.json │ │ │ ├── web3_response.klay_getTransactionReceipt_0x50a7db8f2ac7564dd6ed0841728c5c43882b136c0130fa4b32989f5403d5e3e7.json │ │ │ ├── web3_response.klay_getTransactionReceipt_0xa8d369dd8aac128b08451a2c08ceda10d9735a8e53c8b3a01aa19df51eda5c79.json │ │ │ ├── web3_response.klay_getTransactionReceipt_0xbc22a4cd65003337d6bc70297df4401cab342e710f67769dc13999df92b34306.json │ │ │ ├── web3_response.klay_getTransactionReceipt_0xb93317dd74dd711b01a48debacb672e6c75bf000ee4e23831d85fce99eef576f.json │ │ │ ├── web3_response.klay_getTransactionReceipt_0x5b665efa2da5a58da36af0e854734273247b4af3f38892412c07447ddde2a65a.json │ │ │ ├── web3_response.klay_getTransactionReceipt_0x5778fac21d744201bf0cce8c014207938f55eb9213989086f3f2228acbac4b54.json │ │ │ └── web3_response.klay_getTransactionReceipt_0x5eb066bd544519dd77657983d2daf67183bd9ad2c7c581978746d9a27f6a79f3.json │ ├── test_extract_token_transfers_job │ │ └── logs │ │ │ └── expected_token_transfers.csv │ └── __init__.py ├── __init__.py ├── klaytnetl │ ├── __init__.py │ ├── job │ │ ├── __init__.py │ │ ├── mock_ipfs_client.py │ │ ├── mock_batch_web3_provider.py │ │ ├── mock_web3_provider.py │ │ ├── helpers.py │ │ └── test_extract_token_transfers_job.py │ ├── service │ │ ├── __init__.py │ │ └── test_klaytn_token_transfer_extractor.py │ ├── test_progress_logger.py │ └── test_utils.py └── helpers.py ├── Dockerfile ├── requirements.txt ├── mkdocs.yml ├── docs ├── index.md ├── quickstart.md ├── limitations.md └── exporting-the-blockchain.md ├── schemas ├── aws │ ├── logs.sql │ ├── token_transfers.sql │ ├── parquet │ │ ├── parquet_token_transfers.sql │ │ ├── parquet_blocks.sql │ │ └── parquet_transactions.sql │ ├── contracts.sql │ ├── tokens.sql │ ├── blocks.sql │ ├── transactions.sql │ └── receipts.sql ├── logs.py ├── token_transfers.py └── contracts.py ├── .gitignore ├── .dockerignore ├── LICENSE ├── klaytnetl ├── __init__.py ├── jobs │ ├── __init__.py │ ├── exporters │ │ ├── __init__.py │ │ ├── tokens_item_exporter.py │ │ ├── contracts_item_exporter.py │ │ ├── token_transfers_item_exporter.py │ │ ├── traces_item_exporter.py │ │ ├── raw_traces_item_exporter.py │ │ ├── enrich_traces_item_exporter.py │ │ ├── receipts_and_logs_item_exporter.py │ │ └── blocks_and_transactions_item_exporter.py │ ├── extract_tokens_job.py │ └── extract_token_transfers_job.py ├── misc │ ├── __init__.py │ └── retriable_value_error.py ├── mixin │ ├── __init__.py │ └── enrichable_mixin.py ├── domain │ ├── __init__.py │ └── base.py ├── executors │ ├── __init__.py │ ├── fail_safe_executor.py │ └── bounded_executor.py ├── mappers │ ├── __init__.py │ └── base.py ├── providers │ ├── __init__.py │ ├── rpc.py │ └── auto.py ├── service │ ├── __init__.py │ └── trace_block_service.py ├── __main__.py ├── web3_utils.py ├── atomic_counter.py ├── erc165_abi.py ├── thread_local_proxy.py ├── csv_utils.py ├── cli │ ├── extract_field.py │ ├── get_keccak_hash.py │ ├── filter_items.py │ ├── extract_csv_column.py │ ├── s3_sync.py │ ├── gcs_sync.py │ ├── extract_token_transfers.py │ └── extract_contracts.py └── trace_progress_logger.py ├── klaytnetl.py ├── README.md └── setup.py /blockchainetl/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /blockchainetl/jobs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /blockchainetl/jobs/exporters/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /blockchainetl/jobs/exporters/converters/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @kjhman21 @jeongkyun-oh @yongchand 2 | -------------------------------------------------------------------------------- /tests/resources/test_export_traces_job/block_without_transactions/expected_traces.json: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/resources/test_export_blocks_job/block_without_transactions/expected_transactions.json: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/resources/test_export_tokens_job/erc20_tokens/web3_response.eth_chainId_.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":"0x2019"} -------------------------------------------------------------------------------- /tests/resources/test_export_tokens_job/erc721_tokens/web3_response.eth_chainId_.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":"0x2019"} -------------------------------------------------------------------------------- /tests/resources/test_export_token_transfers_job/block_with_transfers/web3_response.eth_uninstallFilter_0x849659f71ba861ad7a3e6b45e34a5045.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":true} -------------------------------------------------------------------------------- /tests/resources/test_export_traces_job/block_without_transactions/web3_response.debug_traceBlockByNumber_0x1_timeout_1h_tracer_fastcalltracer.json: -------------------------------------------------------------------------------- 1 | { 2 | "jsonrpc": "2.0", 3 | "result": [], 4 | "id": 0 5 | } -------------------------------------------------------------------------------- /tests/resources/test_export_tokens_job/erc1155_tokens/expected_tokens.json: -------------------------------------------------------------------------------- 1 | {"address": "0x4e16e2567dd332d4c44474f8b8d3130b5c311cf7", "symbol": null, "name": null, "decimals": null, "total_supply": null, "block_number": null} 2 | -------------------------------------------------------------------------------- /tests/resources/test_export_tokens_job/erc721_tokens/expected_tokens.json: -------------------------------------------------------------------------------- 1 | {"address": "0xb88168dde0001be8546c70c117ab9e41e28f7164", "symbol": "PNFT", "name": "Pioneer NFT", "decimals": null, "total_supply": 4992, "block_number": null} 2 | -------------------------------------------------------------------------------- /tests/resources/test_export_tokens_job/erc20_tokens/web3_response.eth_call_data_0x18160ddd_to_0x9e481eb17d3c3c07d7a6ab571b4ba8ef432b5cf2_latest.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":"0x0000000000000000000000000000000000000000033b2e3c9fd0803ce8000000"} -------------------------------------------------------------------------------- /tests/resources/test_export_tokens_job/erc20_tokens/web3_response.eth_call_data_0x313ce567_to_0x9e481eb17d3c3c07d7a6ab571b4ba8ef432b5cf2_latest.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":"0x0000000000000000000000000000000000000000000000000000000000000012"} -------------------------------------------------------------------------------- /tests/resources/test_export_tokens_job/erc721_tokens/web3_response.eth_call_data_0x18160ddd_to_0xb88168dde0001be8546c70c117ab9e41e28f7164_latest.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":"0x0000000000000000000000000000000000000000000000000000000000001380"} -------------------------------------------------------------------------------- /tests/resources/test_export_trace_groups_job/trace_groups_raw/expected_tokens.json: -------------------------------------------------------------------------------- 1 | {"address": "0x9b488c2b84263939bfa5c75215a832ac7cff884b", "symbol": "Star", "name": "StarDao", "decimals": 9, "total_supply": 2800000000000, "block_number": 77934351} 2 | -------------------------------------------------------------------------------- /tests/resources/test_export_tokens_job/erc20_tokens/expected_tokens.json: -------------------------------------------------------------------------------- 1 | {"address": "0x9e481eb17d3c3c07d7a6ab571b4ba8ef432b5cf2", "symbol": "MCC", "name": "MyCreditChain", "decimals": 18, "total_supply": 1000000000000000000000000000, "block_number": null} 2 | -------------------------------------------------------------------------------- /tests/resources/test_export_token_transfers_job/block_with_transfers/web3_response.eth_newFilter_fromBlock_0x4d67c2e_toBlock_0x4d67c2e_topics_0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":"0x849659f71ba861ad7a3e6b45e34a5045"} -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7 2 | MAINTAINER Chan Hong 3 | ENV PROJECT_DIR=klaytn-etl 4 | 5 | RUN mkdir /$PROJECT_DIR 6 | WORKDIR /$PROJECT_DIR 7 | COPY . . 8 | RUN pip install --upgrade pip && pip install -e /$PROJECT_DIR/ 9 | 10 | ENTRYPOINT ["python", "klaytnetl"] 11 | -------------------------------------------------------------------------------- /tests/resources/test_export_tokens_job/erc20_tokens/web3_response.eth_call_data_0x06fdde03_to_0x9e481eb17d3c3c07d7a6ab571b4ba8ef432b5cf2_latest.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":"0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000d4d79437265646974436861696e00000000000000000000000000000000000000"} -------------------------------------------------------------------------------- /tests/resources/test_export_tokens_job/erc20_tokens/web3_response.eth_call_data_0x95d89b41_to_0x9e481eb17d3c3c07d7a6ab571b4ba8ef432b5cf2_latest.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":"0x000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000034d43430000000000000000000000000000000000000000000000000000000000"} -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | web3>=5.29,<=5.30 2 | eth-rlp<0.3 3 | eth-utils==1.10 4 | eth-abi==2.1.1 5 | # TODO: This has to be removed when "ModuleNotFoundError: No module named 'eth_utils.toolz'" is fixed at eth-abi 6 | python-dateutil>=2.8.0,<3 7 | click==8.0.4 8 | ethereum-dasm==0.1.4 9 | pytz==2022.1 10 | base58 11 | requests 12 | boto3 13 | google-cloud-storage 14 | mkdocs>=1.3 -------------------------------------------------------------------------------- /tests/resources/test_export_tokens_job/erc721_tokens/web3_response.eth_call_data_0x06fdde03_to_0xb88168dde0001be8546c70c117ab9e41e28f7164_latest.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":"0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000b50696f6e656572204e4654000000000000000000000000000000000000000000"} -------------------------------------------------------------------------------- /tests/resources/test_export_tokens_job/erc721_tokens/web3_response.eth_call_data_0x95d89b41_to_0xb88168dde0001be8546c70c117ab9e41e28f7164_latest.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":"0x00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000004504e465400000000000000000000000000000000000000000000000000000000"} -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: Klaytn ETL 2 | nav: 3 | - Overview: index.md 4 | - Quickstart: quickstart.md 5 | - Exporting the Blockchain: exporting-the-blockchain.md 6 | - References: 7 | - Commands: commands.md 8 | - Schema: schema.md 9 | - Limitations: limitations.md 10 | theme: readthedocs 11 | repo_url: https://github.com/klaytn/klaytn-etl 12 | edit_uri: edit/main/docs -------------------------------------------------------------------------------- /blockchainetl/logging_utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | 4 | def logging_basic_config(filename=None): 5 | format = '%(asctime)s - %(name)s [%(levelname)s] - %(message)s' 6 | if filename is not None: 7 | logging.basicConfig(level=logging.INFO, format=format, filename=filename) 8 | else: 9 | logging.basicConfig(level=logging.INFO, format=format) 10 | 11 | logging.getLogger('ethereum_dasm.evmdasm').setLevel(logging.ERROR) 12 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | Klaytn ETL lets you convert Klaytn blockchain data into convenient formats like JSONs, CSVs and relational databases. 4 | This is a fork of [Ethereum ETL](https://github.com/blockchain-etl/ethereum-etl). 5 | 6 | ## Features 7 | 8 | Easily export: 9 | 10 | * Blocks 11 | * Transactions 12 | * ERC20 / ERC721 / ERC 1155 tokens 13 | * Token transfers 14 | * Receipts 15 | * Logs 16 | * Contracts 17 | * Traces (Internal transactions) 18 | 19 | -------------------------------------------------------------------------------- /schemas/aws/logs.sql: -------------------------------------------------------------------------------- 1 | CREATE EXTERNAL TABLE IF NOT EXISTS logs ( 2 | log_index BIGINT, 3 | transaction_hash STRING, 4 | transaction_index BIGINT, 5 | address STRING, 6 | data STRING, 7 | topics ARRAY, 8 | block_timestamp TIMESTAMP, 9 | transaction_receipt_status BIGINT, 10 | block_number BIGINT, 11 | block_unix_timestamp DOUBLE, 12 | block_hash STRING 13 | ) 14 | PARTITIONED BY (block_date STRING) 15 | ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe' 16 | LOCATION 's3:///export/logs/'; 17 | 18 | MSCK REPAIR TABLE logs; -------------------------------------------------------------------------------- /schemas/aws/token_transfers.sql: -------------------------------------------------------------------------------- 1 | CREATE EXTERNAL TABLE IF NOT EXISTS token_transfers ( 2 | token_address STRING, 3 | from_address STRING, 4 | to_address STRING, 5 | value DECIMAL(38, 0), 6 | transaction_hash STRING, 7 | transaction_index BIGINT, 8 | log_index BIGINT, 9 | block_timestamp TIMESTAMP, 10 | block_unix_timestamp DOUBLE, 11 | transaction_receipt_status BIGINT, 12 | block_number BIGINT, 13 | block_hash STRING 14 | ) 15 | PARTITIONED BY (block_date STRING) 16 | ROW FORMAT SERDE ''org.apache.hive.hcatalog.data.JsonSerDe'' 17 | LOCATION ''s3:///export/token_transfers/''; 18 | 19 | MSCK 20 | REPAIR TABLE token_transfers; -------------------------------------------------------------------------------- /schemas/aws/parquet/parquet_token_transfers.sql: -------------------------------------------------------------------------------- 1 | CREATE EXTERNAL TABLE IF NOT EXISTS parquet_token_transfers ( 2 | token_address STRING, 3 | from_address STRING, 4 | to_address STRING, 5 | value DECIMAL(38, 0), 6 | transaction_hash STRING, 7 | transaction_index BIGINT, 8 | log_index BIGINT, 9 | block_timestamp TIMESTAMP, 10 | block_unix_timestamp DOUBLE, 11 | transaction_receipt_status BIGINT, 12 | block_number BIGINT, 13 | block_hash STRING 14 | ) 15 | PARTITIONED BY (start_block BIGINT, end_block BIGINT) 16 | STORED AS PARQUET 17 | LOCATION 's3:///klaytnetl/parquet/token_transfers'; 18 | 19 | MSCK REPAIR TABLE parquet_token_transfers; -------------------------------------------------------------------------------- /schemas/aws/contracts.sql: -------------------------------------------------------------------------------- 1 | CREATE EXTERNAL TABLE IF NOT EXISTS contracts ( 2 | address STRING, 3 | bytecode STRING, 4 | function_sighashes array, 5 | is_erc20 BOOLEAN, 6 | is_erc721 BOOLEAN, 7 | block_number BIGINT, 8 | block_hash STRING, 9 | block_unix_timestamp DOUBLE, 10 | block_timestamp TIMESTAMP, 11 | transaction_hash STRING, 12 | transaction_index BIGINT, 13 | transaction_receipt_status BIGINT, 14 | trace_index BIGINT, 15 | trace_status BIGINT, 16 | creator_address STRING 17 | ) 18 | PARTITIONED BY (block_date STRING) 19 | ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe' 20 | LOCATION 's3:///export/contracts/'; 21 | 22 | MSCK REPAIR TABLE contracts; -------------------------------------------------------------------------------- /.github/workflows/pytest.yml: -------------------------------------------------------------------------------- 1 | name: Pytest PR 2 | on: 3 | pull_request: 4 | branches: 5 | - main 6 | jobs: 7 | run-test-code: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v3 11 | - name: Set up Python 3.9 12 | uses: actions/setup-python@v4 13 | with: 14 | python-version: '3.9' 15 | cache: 'pip' 16 | - name: Install dependencies 17 | run: | 18 | python -m pip install --upgrade pip 19 | pip install pytest 20 | pip install -r requirements.txt 21 | - name: Test with pytest 22 | env: 23 | KLAYTN_ETL_RUN_SLOW_TESTS: True 24 | run: | 25 | pytest 26 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish pypi package 2 | on: 3 | push: 4 | tags: 5 | - v[0-9]+.[0-9]+.[0-9]+ 6 | 7 | jobs: 8 | publish: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v3 12 | - name: Set up Python 3.10 13 | uses: actions/setup-python@v3 14 | with: 15 | python-version: '3.10' 16 | - name: Build python package 17 | run: | 18 | echo ${{ github.ref_name }} > VERSION 19 | python -m pip install build wheel twine 20 | python setup.py bdist_wheel 21 | - name: pypi-publish 22 | uses: pypa/gh-action-pypi-publish@release/v1 23 | with: 24 | password: ${{ secrets.PYPI_API_TOKEN }} 25 | 26 | -------------------------------------------------------------------------------- /schemas/aws/tokens.sql: -------------------------------------------------------------------------------- 1 | CREATE EXTERNAL TABLE IF NOT EXISTS tokens ( 2 | address STRING, 3 | symbol STRING, 4 | name STRING, 5 | decimals BIGINT, 6 | total_supply DECIMAL(38, 0), 7 | block_number BIGINT, 8 | block_hash STRING, 9 | function_sighashes ARRAY, 10 | is_erc20 boolean, 11 | is_erc721 boolean, 12 | block_unix_timestamp DOUBLE, 13 | block_timestamp TIMESTAMP, 14 | transaction_hash STRING, 15 | transaction_index BIGINT, 16 | transaction_receipt_status BIGINT, 17 | trace_index BIGINT, 18 | trace_status BIGINT, 19 | creator_address STRING 20 | ) 21 | PARTITIONED BY (block_date STRING) 22 | ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe' 23 | LOCATION 's3:///export/tokens/'; 24 | 25 | MSCK REPAIR TABLE tokens; -------------------------------------------------------------------------------- /schemas/aws/blocks.sql: -------------------------------------------------------------------------------- 1 | CREATE EXTERNAL TABLE IF NOT EXISTS blocks ( 2 | number BIGINT, 3 | hash STRING, 4 | parent_hash STRING, 5 | logs_bloom STRING, 6 | transactions_root STRING, 7 | state_root STRING, 8 | receipts_root STRING, 9 | block_score DECIMAL(38,0) , 10 | total_block_score DECIMAL(38,0), 11 | size BIGINT, 12 | extra_data STRING, 13 | gas_used BIGINT , 14 | timestamp TIMESTAMP, 15 | unix_timestamp DOUBLE, 16 | transaction_count BIGINT, 17 | governance_data STRING, 18 | vote_data STRING, 19 | proposer STRING, 20 | committee array, 21 | reward_address STRING, 22 | base_fee_per_gas DECIMAL(38, 0) 23 | ) 24 | PARTITIONED BY (block_date STRING) 25 | ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe' 26 | LOCATION 's3:///export/blocks/'; 27 | 28 | MSCK REPAIR TABLE blocks; -------------------------------------------------------------------------------- /docs/quickstart.md: -------------------------------------------------------------------------------- 1 | # Quickstart 2 | Install Klaytn ETL: 3 | 4 | ```bash 5 | pip3 install klaytn-etl-cli 6 | ``` 7 | 8 | Export blocks and transactions 9 | 10 | ```bash 11 | > klaytnetl export_blocks_and_transactions --start-block 0 --end-block 5000 \ 12 | --blocks-output blocks.json --transactions-output transactions.json 13 | ``` 14 | 15 | Export ERC20, ERC721, ERC1155 transfers 16 | 17 | ```bash 18 | > klaytnetl export_token_transfers --start-block 0 --end-block 5000 \ 19 | --output token_transfers.json 20 | ``` 21 | 22 | Export traces 23 | 24 | ```bash 25 | > klaytnetl export_traces --start-block 0 --end-block 5000 \ 26 | --output traces.json 27 | ``` 28 | 29 | Find other commands [here](commands.md). 30 | 31 | For the latest version, check out the repo and call 32 | ```bash 33 | > pip3 install -e . 34 | > python3 klaytnetl.py 35 | ``` 36 | -------------------------------------------------------------------------------- /schemas/aws/parquet/parquet_blocks.sql: -------------------------------------------------------------------------------- 1 | CREATE EXTERNAL TABLE IF NOT EXISTS parquet_blocks ( 2 | number BIGINT, 3 | hash STRING, 4 | parent_hash STRING, 5 | logs_bloom STRING, 6 | transactions_root STRING, 7 | state_root STRING, 8 | receipts_root STRING, 9 | block_score DECIMAL(38,0) , 10 | total_block_score DECIMAL(38,0), 11 | size BIGINT, 12 | extra_data STRING, 13 | gas_used BIGINT , 14 | timestamp TIMESTAMP, 15 | unix_timestamp DOUBLE, 16 | transaction_count BIGINT, 17 | governance_data STRING, 18 | vote_data STRING, 19 | proposer STRING, 20 | committee array. 21 | reward_address STRING, 22 | base_fee_per_gas DECIMAL(38, 0) 23 | ) 24 | PARTITIONED BY (start_block BIGINT, end_block BIGINT) 25 | STORED AS PARQUET 26 | LOCATION 's3:///klaytnetl/parquet/blocks'; 27 | 28 | MSCK REPAIR TABLE parquet_blocks; -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # custom 2 | out/ 3 | scripts/ 4 | test_out/ 5 | schemas/poc/ 6 | 7 | # Byte-compiled / optimized / DLL files 8 | __pycache__/ 9 | *.py[cod] 10 | *$py.class 11 | 12 | # C extensions 13 | *.so 14 | 15 | # Distribution / packaging 16 | .Python 17 | env/ 18 | build/ 19 | develop-eggs/ 20 | dist/ 21 | downloads/ 22 | eggs/ 23 | .eggs/ 24 | lib/ 25 | lib64/ 26 | parts/ 27 | sdist/ 28 | var/ 29 | wheels/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # dotenv 50 | .env 51 | 52 | # virtualenv 53 | .venv 54 | venv/ 55 | ENV/ 56 | 57 | # IDE 58 | .idea 59 | 60 | # DS Store 61 | .DS_Store 62 | */.DS_Store -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .* 2 | last_synced_block.txt 3 | pid.txt 4 | output 5 | 6 | # custom 7 | tests/ 8 | out/ 9 | scripts/ 10 | test_out/ 11 | schemas/poc/ 12 | 13 | # Byte-compiled / optimized / DLL files 14 | __pycache__/ 15 | *.py[cod] 16 | *$py.class 17 | 18 | # C extensions 19 | *.so 20 | 21 | # Distribution / packaging 22 | .Python 23 | env/ 24 | build/ 25 | develop-eggs/ 26 | dist/ 27 | downloads/ 28 | eggs/ 29 | .eggs/ 30 | lib/ 31 | lib64/ 32 | parts/ 33 | sdist/ 34 | var/ 35 | wheels/ 36 | *.egg-info/ 37 | .installed.cfg 38 | *.egg 39 | 40 | # Installer logs 41 | pip-log.txt 42 | pip-delete-this-directory.txt 43 | 44 | # Unit test / coverage reports 45 | htmlcov/ 46 | .tox/ 47 | .coverage 48 | .coverage.* 49 | .cache 50 | nosetests.xml 51 | coverage.xml 52 | *.cover 53 | .hypothesis/ 54 | 55 | # dotenv 56 | .env 57 | 58 | # virtualenv 59 | .venv 60 | venv/ 61 | ENV/ 62 | -------------------------------------------------------------------------------- /docs/limitations.md: -------------------------------------------------------------------------------- 1 | # Limitation 2 | 3 | - In case the contract is a proxy, which forwards all calls to a delegate, interface detection doesn’t work, 4 | which means `is_erc20`/`is_erc721`/`is_erc1155` will always be false for proxy contracts and they will be missing in the `tokens` 5 | table. 6 | - The metadata methods (`symbol`, `name`, `decimals`, `total_supply`) for ERC20 are optional, so around 10% of the 7 | contracts are missing this data. 8 | - `token_transfers.value`, `tokens.decimals` and `tokens.total_supply` have type `STRING` in BigQuery tables, 9 | because numeric types there can't handle 32-byte integers. You should use 10 | `cast(value as FLOAT64)` (possible loss of precision) or 11 | `safe_cast(value as NUMERIC)` (possible overflow) to convert to numbers. 12 | - The contracts that don't implement `decimals()` function but have the 13 | [fallback function](https://solidity.readthedocs.io/en/v0.4.21/contracts.html#fallback-function) that returns a `boolean` 14 | will have `0` or `1` in the `decimals` column in the CSVs. -------------------------------------------------------------------------------- /schemas/aws/transactions.sql: -------------------------------------------------------------------------------- 1 | CREATE EXTERNAL TABLE IF NOT EXISTS transactions ( 2 | hash STRING, 3 | nonce BIGINT, 4 | block_hash STRING, 5 | block_number BIGINT, 6 | transaction_index BIGINT, 7 | from_address STRING, 8 | to_address STRING, 9 | value DECIMAL(38, 0), 10 | gas DECIMAL(38, 0), 11 | gas_price DECIMAL(38, 0), 12 | input STRING, 13 | fee_payer STRING, 14 | fee_payer_signatures ARRAY>, 15 | fee_ratio BIGINT, 16 | sender_tx_hash STRING, 17 | signatures ARRAY>, 18 | tx_type STRING, 19 | tx_type_int BIGINT, 20 | max_priority_fee_per_gas DECIMAL(38, 0), 21 | max_fee_per_gas DECIMAL(38,0), 22 | block_timestamp TIMESTAMP, 23 | block_unix_timestamp DOUBLE, 24 | receipt_gas_used BIGINT, 25 | receipt_contract_address STRING, 26 | receipt_status BIGINT 27 | ) 28 | PARTITIONED BY (block_date STRING) 29 | ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe' 30 | LOCATION 's3:///export/transactions/'; 31 | 32 | MSCK REPAIR TABLE transactions; -------------------------------------------------------------------------------- /schemas/aws/parquet/parquet_transactions.sql: -------------------------------------------------------------------------------- 1 | CREATE EXTERNAL TABLE IF NOT EXISTS parquet_transactions ( 2 | hash STRING, 3 | nonce BIGINT, 4 | block_hash STRING, 5 | block_number BIGINT, 6 | transaction_index BIGINT, 7 | from_address STRING, 8 | to_address STRING, 9 | value DECIMAL(38, 0), 10 | gas DECIMAL(38, 0), 11 | gas_price DECIMAL(38, 0), 12 | input STRING, 13 | fee_payer STRING, 14 | fee_payer_signatures ARRAY>, 15 | fee_ratio BIGINT, 16 | sender_tx_hash STRING, 17 | signatures ARRAY>, 18 | tx_type STRING, 19 | tx_type_int BIGINT, 20 | max_priority_fee_per_gas DECIMAL(38, 0), 21 | max_fee_per_gas DECIMAL(38,0), 22 | block_timestamp TIMESTAMP, 23 | block_unix_timestamp DOUBLE, 24 | receipt_gas_used BIGINT, 25 | receipt_contract_address STRING, 26 | receipt_status BIGINT 27 | ) 28 | PARTITIONED BY (start_block BIGINT, end_block BIGINT) 29 | STORED AS PARQUET 30 | LOCATION 's3:///klaytnetl/parquet/transactions'; 31 | 32 | MSCK REPAIR TABLE parquet_transactions; -------------------------------------------------------------------------------- /tests/resources/test_export_trace_groups_job/trace_groups_enrich/expected_tokens.json: -------------------------------------------------------------------------------- 1 | {"address": "0x9b488c2b84263939bfa5c75215a832ac7cff884b", "symbol": "Star", "name": "StarDao", "decimals": 9, "total_supply": 2800000000000, "function_sighashes": ["0x06fdde03", "0x095ea7b3", "0x18160ddd", "0x23b872dd", "0x30adf81f", "0x313ce567", "0x3644e515", "0x39509351", "0x40c10f19", "0x42966c68", "0x616c6c65", "0x655a6572", "0x6817031b", "0x70a08231", "0x715018a6", "0x79cc6790", "0x7ecebe00", "0x8da5cb5b", "0x95d89b41", "0xa22b35ce", "0xa457c2d7", "0xa9059cbb", "0xd505accf", "0xdd62ed3e", "0xf2fde38b", "0xfbfa77cf"], "is_erc20": true, "is_erc721": false, "is_erc1155": false, "block_number": 77934351, "block_hash": "0x0efcfaced5d13ab6c14ed865f916be3d8031f0af6b264fbfc2352b7e69dfbb82", "block_timestamp": "2021-12-17T11:23:03.065000+00:00", "block_unix_timestamp": 1639740183.065, "transaction_hash": "0xdc13f88ddb6966bb56b793ac230489e2f23d4eb5002f022f4cc235905e785dfb", "transaction_index": 7, "transaction_receipt_status": 1, "trace_index": 168, "trace_status": 1, "creator_address": "0x9fd08df2775bb3b5dfa740fd3342bc3bec0095b8"} 2 | -------------------------------------------------------------------------------- /schemas/aws/receipts.sql: -------------------------------------------------------------------------------- 1 | CREATE EXTERNAL TABLE IF NOT EXISTS receipts ( 2 | transaction_hash STRING, 3 | transaction_index BIGINT, 4 | block_hash STRING, 5 | block_number BIGINT, 6 | gas DECIMAL(38, 0), 7 | gas_price DECIMAL(38, 0), 8 | gas_used DECIMAL(38, 0), 9 | effective_gas_price DECIMAL(38, 0), 10 | contract_address STRING, 11 | logs_bloom STRING, 12 | nonce BIGINT, 13 | fee_payer STRING, 14 | fee_payer_signatures ARRAY>, 15 | fee_ratio BIGINT, 16 | code_format STRING, 17 | human_readable BOOLEAN, 18 | tx_error STRING, 19 | key STRING, 20 | input_data STRING, 21 | from_address STRING, 22 | to_address STRING, 23 | type_name STRING, 24 | type_int BIGINT, 25 | sender_tx_hash STRING, 26 | signatures ARRAY>, 27 | value DECIMAL(38, 0), 28 | chain_id BIGINT, 29 | max_priority_fee_per_gas DECIMAL(38, 0), 30 | max_fee_per_gas DECIMAL(38,0) 31 | ) 32 | PARTITIONED BY (block_date STRING) 33 | ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe' 34 | LOCATION 's3:///export/receipts/'; 35 | 36 | MSCK REPAIR TABLE receipts; -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Modifications Copyright (c) klaytn authors 4 | Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com, https://twitter.com/EvgeMedvedev 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | -------------------------------------------------------------------------------- /klaytnetl/__init__.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | -------------------------------------------------------------------------------- /klaytnetl/jobs/__init__.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | -------------------------------------------------------------------------------- /klaytnetl/misc/__init__.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | -------------------------------------------------------------------------------- /klaytnetl/mixin/__init__.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | -------------------------------------------------------------------------------- /tests/klaytnetl/__init__.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | -------------------------------------------------------------------------------- /klaytnetl/domain/__init__.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | -------------------------------------------------------------------------------- /klaytnetl/executors/__init__.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | -------------------------------------------------------------------------------- /klaytnetl/mappers/__init__.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | -------------------------------------------------------------------------------- /klaytnetl/providers/__init__.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | -------------------------------------------------------------------------------- /klaytnetl/service/__init__.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | -------------------------------------------------------------------------------- /tests/klaytnetl/job/__init__.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | -------------------------------------------------------------------------------- /klaytnetl/jobs/exporters/__init__.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | -------------------------------------------------------------------------------- /tests/klaytnetl/service/__init__.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | -------------------------------------------------------------------------------- /klaytnetl.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | from klaytnetl.cli import cli 26 | 27 | cli() 28 | -------------------------------------------------------------------------------- /klaytnetl/domain/base.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | class BaseDomain(object): 26 | pass 27 | -------------------------------------------------------------------------------- /klaytnetl/__main__.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | from klaytnetl.cli import cli 26 | 27 | cli() 28 | -------------------------------------------------------------------------------- /klaytnetl/misc/retriable_value_error.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | class RetriableValueError(ValueError): 26 | pass 27 | -------------------------------------------------------------------------------- /docs/exporting-the-blockchain.md: -------------------------------------------------------------------------------- 1 | ## Exporting the Blockchain 2 | 3 | 1. Install python 3.7.2+: [https://www.python.org/downloads/](https://www.python.org/downloads/) 4 | 5 | 1. Launch an [endpoint node](https://docs.klaytn.foundation/getting-started/quick-start/launch-an-en) or use [pre-existing endpoint](https://docs.klaytn.foundation/dapp/json-rpc/public-en) 6 | 7 | 1. Install Klaytn ETL: `> pip3 install klaytn-etl-cli` 8 | 9 | 1. Export all: 10 | 11 | ```bash 12 | > klaytnetl export_all --help 13 | > klaytnetl export_all -s 0 -e 5999999 -b 100000 -p https://cypress.fandom.finance/archive -o output 14 | ``` 15 | 16 | In case `klaytnetl` command is not available in PATH, use `python3 -m klaytnetl` instead. 17 | 18 | The result will be in the `output` subdirectory, partitioned in Hive style: 19 | ```bash 20 | output/blocks/start_block=00000000/end_block=00099999/blocks_00000000_00099999.csv 21 | output/blocks/start_block=00100000/end_block=00199999/blocks_00100000_00199999.csv 22 | ... 23 | output/transactions/start_block=00000000/end_block=00099999/transactions_00000000_00099999.csv 24 | ... 25 | output/token_transfers/start_block=00000000/end_block=00099999/token_transfers_00000000_00099999.csv 26 | ... 27 | ``` 28 | 29 | Should work on Linux, Mac, Windows. 30 | Since `debug_traceBlockByNumber` takes a long time, please cautious when running anything related to trace. 31 | -------------------------------------------------------------------------------- /tests/resources/test_export_receipts_job/receipts/web3_response.klay_getTransactionReceipt_0x19485bb473e7d90af7d3a67d6631f91f5abf6a6a1e24ff24a3dfe30c5028e168.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":{"blockHash":"0x55faf2d88e2860befa23414932da8f8b37fee4ea927133619c74be3ff8e997b7","blockNumber":"0x5cc8408","contractAddress":null,"effectiveGasPrice":"0x3a35294400","from":"0x8da74ba3db7b440264421373d2cbfca2b10e2e3b","gas":"0x3d090","gasPrice":"0x3a35294400","gasUsed":"0x5208","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","nonce":"0x302","senderTxHash":"0x19485bb473e7d90af7d3a67d6631f91f5abf6a6a1e24ff24a3dfe30c5028e168","signatures":[{"V":"0x4056","R":"0xb320118b4f4a0fb28988aea7bd181b6bfcd796fe65aabd651c6a7c02481d0893","S":"0x348c22ee62789aaa0c11b97a9a8770327dd9f230701fdcea730a80ee5ff6b009"}],"status":"0x1","transactionHash":"0x19485bb473e7d90af7d3a67d6631f91f5abf6a6a1e24ff24a3dfe30c5028e168","transactionIndex":"0x15","type":"TxTypeCancel","typeInt":56}} -------------------------------------------------------------------------------- /klaytnetl/mappers/base.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | class BaseMapper(object): 26 | def __init__(self, **kwargs): 27 | super(BaseMapper, self).__init__(**kwargs) 28 | -------------------------------------------------------------------------------- /tests/resources/test_export_receipts_job/receipts/web3_response.klay_getTransactionReceipt_0x16fe9780d7d159958425877870e0a9db3187392b2386b1e81463ae03b8d92512.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":{"blockHash":"0x902de6790208ccdcaea3f2fa804a2554da2624699f2b98c35adeebec25dab35c","blockNumber":"0x5d973dc","contractAddress":null,"effectiveGasPrice":"0x3a35294400","from":"0x148d2eda81fc81ca396bfe62f6cce538fcfac21d","gas":"0x5208","gasPrice":"0x3a35294400","gasUsed":"0x5208","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","nonce":"0x244","senderTxHash":"0x16fe9780d7d159958425877870e0a9db3187392b2386b1e81463ae03b8d92512","signatures":[{"V":"0x4056","R":"0xdcc405e50bee73701ee56c00136d56dc4f9cfa46a7bb545ff403dc2a423e30bb","S":"0x721fbaed38220fc7df16d3487419d8bbcb7ce7bea44ae3f5f0e8bf9d52ec793a"}],"status":"0x1","to":"0xd9c68b6e6ca0fd44626e55591779b423f854dc1a","transactionHash":"0x16fe9780d7d159958425877870e0a9db3187392b2386b1e81463ae03b8d92512","transactionIndex":"0x4","type":"TxTypeValueTransfer","typeInt":8,"value":"0x233c8fe42703e80000"}} -------------------------------------------------------------------------------- /tests/resources/test_export_receipts_job/receipts/web3_response.klay_getTransactionReceipt_0x19613c8cf5a711cc3701dc5346ad095171150601abfcaf8e74ecbd536d4e4834.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":{"blockHash":"0xaa35335c0f9cec47dc17d26f18188885f2d429769eacd830b692e5b136bb9308","blockNumber":"0x5d14953","contractAddress":null,"effectiveGasPrice":"0x3a35294400","from":"0x0bbb1360f23799d4857b3d267198b334f284914f","gas":"0x2dc6c0","gasPrice":"0x3a35294400","gasUsed":"0xa028","key":"0x02a102a01616972cd1bf0e5fb6b5478a1d933ab92707a1c12a55a0069dc9bafed17774","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","nonce":"0xd1","senderTxHash":"0x19613c8cf5a711cc3701dc5346ad095171150601abfcaf8e74ecbd536d4e4834","signatures":[{"V":"0x4055","R":"0xac7a96b13ce69957fde9a944bb8469e0da336278261de1041c7edb3a6e4c8ee7","S":"0x339d7bfefdce0e016a87bb47df9b2df92ea05f305e46540e2b1ce623ce99eb7d"}],"status":"0x1","transactionHash":"0x19613c8cf5a711cc3701dc5346ad095171150601abfcaf8e74ecbd536d4e4834","transactionIndex":"0x0","type":"TxTypeAccountUpdate","typeInt":32}} -------------------------------------------------------------------------------- /tests/resources/test_extract_token_transfers_job/logs/expected_token_transfers.csv: -------------------------------------------------------------------------------- 1 | token_address,from_address,to_address,value,transaction_hash,log_index,block_number 2 | 0x3cb6be2fc6677a63cb52b07aed523f93f5a06cb4,0xece13969d1c808d16deac3f8c0314a57e650e0f9,0x51f6946c3de7e2cd95ae364589356ae24e22638f,115000000,0x91b77a5a1a7956cf04f76bc0f7cc72fd97d894146638d86502d0534f480a8b7d,0,81165358 3 | 0x9e481eb17d3c3c07d7a6ab571b4ba8ef432b5cf2,0xe5003b373185d6e3c0319614e9ce9f812ae5e7f9,0x3d58cc3633b2b172c65100ef00083d49582bcb4d,2000000000000000000,0x77ca51b372dfba47cbccc6866ef301a4489bf8e19baafb78a5a24affbe75176b,1,81165358 4 | 0x5096db80b21ef45230c9e423c373f1fc9c0198dd,0xd5a3d184533bdb0383bb387a154b04b58b15704b,0xe549a25d0d3e44bb3e2a02e6c15ff6dc637dc33d,605951500000000000,0x4b1c1ab197eeda00677a7c06359a9e4d4f26e707c5b97059b383d706757c3154,2,81165358 5 | 0x5096db80b21ef45230c9e423c373f1fc9c0198dd,0x3cb3890004e8f5a6885c1d8d359e8c2946b504f3,0xdfcaa7fed1e1f54dde2bf9adb0f6545fe4eab881,361570900000000000,0xe0e6847a72686f85bba4041d8f86e8ee7df06cd5193755eba7cdc6551a5984fb,3,81165358 6 | 0x5096db80b21ef45230c9e423c373f1fc9c0198dd,0x0ab7f556840bbace9188d843722346eb6115a981,0xe549a25d0d3e44bb3e2a02e6c15ff6dc637dc33d,605951500000000000,0xf913c2db10c34f4c62538f255cc83c3ee16b7b245c34bdd6d95c5233fc0cb45e,5,81165358 7 | 0x5096db80b21ef45230c9e423c373f1fc9c0198dd,0x87554f476813960207f43dacb73d4fc7aa0e7e0a,0xe549a25d0d3e44bb3e2a02e6c15ff6dc637dc33d,605951500000000000,0xd6d5e6f4b0e0e34eae62494993a4d47d987759036f93328bddf2943b029480fc,6,81165358 8 | -------------------------------------------------------------------------------- /klaytnetl/web3_utils.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | from web3 import Web3 26 | from web3.middleware import geth_poa_middleware 27 | 28 | 29 | def build_web3(provider): 30 | w3 = Web3(provider) 31 | w3.middleware_onion.inject(geth_poa_middleware, layer=0) 32 | return w3 33 | -------------------------------------------------------------------------------- /tests/resources/test_export_receipts_job/receipts/web3_response.klay_getTransactionReceipt_0xa4d4c825506b9efaf59bc596c9547fc1a4587ef45bac0aed730ad718434130c6.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":{"blockHash":"0x359be9ef8e0fdddd9cb1ad839250f4fac3969d3ffa53751aa29e75c1cfbbba6e","blockNumber":"0x5d964c2","contractAddress":null,"effectiveGasPrice":"0x3a35294400","from":"0x12ee4bd8337aefccfc327e11458046c9d9816f06","gas":"0xf4240","gasPrice":"0x3a35294400","gasUsed":"0x53fc","input":"0x3078313233","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","nonce":"0x3955","senderTxHash":"0xa4d4c825506b9efaf59bc596c9547fc1a4587ef45bac0aed730ad718434130c6","signatures":[{"V":"0x4056","R":"0x7f9ead00728e2862ba3b464ec3e366c48523634a8f192dbffd657342e38a3aae","S":"0x18512d045f899ef1e62304e93d2029236e082fdc781290ee627de7d4ebba336c"}],"status":"0x1","to":"0xf85c6d2c9320a0a35b5b37cbd3f1b103bce242a3","transactionHash":"0xa4d4c825506b9efaf59bc596c9547fc1a4587ef45bac0aed730ad718434130c6","transactionIndex":"0x0","type":"TxTypeValueTransferMemo","typeInt":16,"value":"0x4c4855d59f3dd800"}} -------------------------------------------------------------------------------- /tests/resources/test_export_token_transfers_job/block_with_transfers/expected_token_transfers.csv: -------------------------------------------------------------------------------- 1 | token_address,from_address,to_address,value,transaction_hash,log_index,block_number 2 | 0x3cb6be2fc6677a63cb52b07aed523f93f5a06cb4,0xece13969d1c808d16deac3f8c0314a57e650e0f9,0x51f6946c3de7e2cd95ae364589356ae24e22638f,115000000,0x91b77a5a1a7956cf04f76bc0f7cc72fd97d894146638d86502d0534f480a8b7d,0,81165358 3 | 0x9e481eb17d3c3c07d7a6ab571b4ba8ef432b5cf2,0xe5003b373185d6e3c0319614e9ce9f812ae5e7f9,0x3d58cc3633b2b172c65100ef00083d49582bcb4d,2000000000000000000,0x77ca51b372dfba47cbccc6866ef301a4489bf8e19baafb78a5a24affbe75176b,1,81165358 4 | 0x5096db80b21ef45230c9e423c373f1fc9c0198dd,0xd5a3d184533bdb0383bb387a154b04b58b15704b,0xe549a25d0d3e44bb3e2a02e6c15ff6dc637dc33d,605951500000000000,0x4b1c1ab197eeda00677a7c06359a9e4d4f26e707c5b97059b383d706757c3154,2,81165358 5 | 0x5096db80b21ef45230c9e423c373f1fc9c0198dd,0x3cb3890004e8f5a6885c1d8d359e8c2946b504f3,0xdfcaa7fed1e1f54dde2bf9adb0f6545fe4eab881,361570900000000000,0xe0e6847a72686f85bba4041d8f86e8ee7df06cd5193755eba7cdc6551a5984fb,3,81165358 6 | 0x5096db80b21ef45230c9e423c373f1fc9c0198dd,0x0ab7f556840bbace9188d843722346eb6115a981,0xe549a25d0d3e44bb3e2a02e6c15ff6dc637dc33d,605951500000000000,0xf913c2db10c34f4c62538f255cc83c3ee16b7b245c34bdd6d95c5233fc0cb45e,5,81165358 7 | 0x5096db80b21ef45230c9e423c373f1fc9c0198dd,0x87554f476813960207f43dacb73d4fc7aa0e7e0a,0xe549a25d0d3e44bb3e2a02e6c15ff6dc637dc33d,605951500000000000,0xd6d5e6f4b0e0e34eae62494993a4d47d987759036f93328bddf2943b029480fc,6,81165358 8 | -------------------------------------------------------------------------------- /blockchainetl/jobs/base_job.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | 24 | class BaseJob(object): 25 | def run(self): 26 | try: 27 | self._start() 28 | self._export() 29 | finally: 30 | self._end() 31 | 32 | def _start(self): 33 | pass 34 | 35 | def _export(self): 36 | pass 37 | 38 | def _end(self): 39 | pass 40 | -------------------------------------------------------------------------------- /blockchainetl/jobs/exporters/console_item_exporter.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | import json 24 | 25 | 26 | class ConsoleItemExporter: 27 | def open(self): 28 | pass 29 | 30 | def export_items(self, items): 31 | for item in items: 32 | self.export_item(item) 33 | 34 | def export_item(self, item): 35 | print(json.dumps(item)) 36 | 37 | def close(self): 38 | pass 39 | -------------------------------------------------------------------------------- /blockchainetl/atomic_counter.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | import itertools 24 | 25 | 26 | # https://stackoverflow.com/a/27062830/1580227 27 | class AtomicCounter: 28 | def __init__(self): 29 | self._counter = itertools.count() 30 | # init to 0 31 | next(self._counter) 32 | 33 | def increment(self, increment=1): 34 | assert increment > 0 35 | return [next(self._counter) for _ in range(0, increment)][-1] 36 | -------------------------------------------------------------------------------- /tests/resources/test_export_receipts_job/receipts/web3_response.klay_getTransactionReceipt_0x9b23a9ea174552fa158b41fab89be404b6b29aa5f24c2217e94625cf8a1c9a54.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":{"blockHash":"0xd21c2b85900e9a67735b2fd7781b2e2aa7ae0db9b8245c3f9ca7f84123db2b19","blockNumber":"0x50305ba","contractAddress":null,"effectiveGasPrice":"0x5d21dba00","feePayer":"0x3d2e3250fa213574928a22a19c38f24d952ab57d","feePayerSignatures":[{"V":"0x4055","R":"0x6c807ef34068d19c17a842ac8b6fba00dcbe5d755484ea6777cc0a824e386fbf","S":"0x3974ed16aa6868166b5c60e7b226878eda63d8f2de9d4a394e089d06bac2da5c"}],"from":"0x3d2e3250fa213574928a22a19c38f24d952ab57d","gas":"0x4c4b400","gasPrice":"0x5d21dba00","gasUsed":"0x7918","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","nonce":"0x14e8","senderTxHash":"0x339504a168241cb948e4cb89658f158b0863b9530eca669976ae62758b81ef56","signatures":[{"V":"0x4055","R":"0xc5cf62b09e500346a57b98abd2da33bd3a54787645c2762c84c60bc52372801","S":"0x3364959824cb0562ed2ce81e41fabdec03399b61d892f8e231e02d34c1243c9b"}],"status":"0x1","transactionHash":"0x9b23a9ea174552fa158b41fab89be404b6b29aa5f24c2217e94625cf8a1c9a54","transactionIndex":"0x7","type":"TxTypeFeeDelegatedCancel","typeInt":57}} -------------------------------------------------------------------------------- /klaytnetl/atomic_counter.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | import itertools 26 | 27 | 28 | # https://stackoverflow.com/a/27062830/1580227 29 | class AtomicCounter: 30 | def __init__(self): 31 | self._counter = itertools.count() 32 | # init to 0 33 | next(self._counter) 34 | 35 | def increment(self, increment=1): 36 | assert increment > 0 37 | return [next(self._counter) for _ in range(0, increment)][-1] 38 | -------------------------------------------------------------------------------- /tests/resources/__init__.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | import os 26 | 27 | 28 | def read_resource(groups, file_name): 29 | current_file_dir = os.path.dirname(__file__) 30 | fixture_file_name = os.path.join(current_file_dir, *groups, file_name) 31 | 32 | if not os.path.exists(fixture_file_name): 33 | raise ValueError("File does not exist: " + fixture_file_name) 34 | 35 | with open(fixture_file_name) as file_handle: 36 | return file_handle.read() 37 | -------------------------------------------------------------------------------- /tests/resources/test_export_receipts_job/receipts/web3_response.klay_getTransactionReceipt_0x090a5a01702402a09481eb7b3c5a895cbb67aa6a6bd2372c334cb8e7ce337571.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":{"blockHash":"0xeb0e82b69f7953525907ecd542053da97ec3c898b72aeaeb92e4ef4387d10d07","blockNumber":"0x5d96ea6","contractAddress":null,"effectiveGasPrice":"0x3a35294400","feePayer":"0x10f0eaa52a474f0605afbb8c17bb9189e6ed006c","feePayerSignatures":[{"V":"0x4055","R":"0x714a2540603efedff1a3a3c322827070c28a60c163b8a5afeddd8d53be8e3674","S":"0x1c8e07c69ded3370c1a8443a84e573f21e903675e28ba0a4109948ca4e482093"}],"from":"0x305ed5c44c4bc5c59835770a476d98ae50766ca7","gas":"0x3d090","gasPrice":"0x3a35294400","gasUsed":"0xc738","key":"0x02a103d899fb865ccf7d621af496f856314602dbc4b7da99f6ccac562c781e6e276f78","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","nonce":"0x0","senderTxHash":"0xaa476125c9977aad98be1fde1346d5aeeb1fb83363aadfec7df80635afc0dd15","signatures":[{"V":"0x4056","R":"0xfab852b6a1d2359c42cd584ce593cb759f6405ea1fa8925c90381927535f10e","S":"0x7f5a24eded7d0689d96543ef4a85a1a9d3c0655b8037e8016f1f37ef8f291282"}],"status":"0x1","transactionHash":"0x090a5a01702402a09481eb7b3c5a895cbb67aa6a6bd2372c334cb8e7ce337571","transactionIndex":"0x2","type":"TxTypeFeeDelegatedAccountUpdate","typeInt":33}} -------------------------------------------------------------------------------- /tests/resources/test_export_receipts_job/receipts/web3_response.klay_getTransactionReceipt_0xd6960970e0d741b37e2832d03ed4f6a935862fe9cea584e116ec3964205a654f.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":{"blockHash":"0xbdbceacb9d28cdb2042f598078a907112f0b069be79a5ad183b4a907b255fd33","blockNumber":"0x5d973d4","contractAddress":null,"effectiveGasPrice":"0x3a35294400","feePayer":"0xa79fd5c61d46256f11d704f9001c1836885e3aad","feePayerSignatures":[{"V":"0x4055","R":"0xb106c9cb3b60fb7dc4e2bad7eda15a4d76e59b94a571f30b926542c268d29a45","S":"0x7968fb3455fd2eb72a320c01e684e0b6fc721f6ac9044d4703460dfb27a664ba"}],"from":"0xe265c36401063fd8d85ec92ce725ea214f764018","gas":"0x1312d00","gasPrice":"0x3a35294400","gasUsed":"0x7918","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","nonce":"0xe","senderTxHash":"0x5426da0261f48d3a35cc7387b8b15993a3b7e3f7a166640f1fa4da0cc452df5b","signatures":[{"V":"0x4055","R":"0x49730ec4ae2f0dad5d05d62b7398b9a531603a84778bf184a86a3b59f8ad9aa4","S":"0x4f16f681d33739780bed3f2ecc508625c013365a75ffdd8a2fc444d45d5f440c"}],"status":"0x1","to":"0xa54b332d1420b9eda6decee41e688a5de456df73","transactionHash":"0xd6960970e0d741b37e2832d03ed4f6a935862fe9cea584e116ec3964205a654f","transactionIndex":"0x3","type":"TxTypeFeeDelegatedValueTransfer","typeInt":9,"value":"0x3a584641aaa338000"}} -------------------------------------------------------------------------------- /tests/resources/test_export_receipts_job/receipts/web3_response.klay_getTransactionReceipt_0x0d0cc98bb9460a70cb649c221e869d186b5b06048be750bc5a997c96d6d2071d.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":{"blockHash":"0xdcb94005c4156f864cb895214574f77a280a122c34912aea96ce0fa686612254","blockNumber":"0x496727a","contractAddress":null,"effectiveGasPrice":"0x5d21dba00","feePayer":"0x08260736c18bd8612bee2b21beedf4e97c0bc6b9","feePayerSignatures":[{"V":"0x4055","R":"0x33d7ef23afca8169a6ebdfd3ccb514dbe7ea6d1aa19d21960683e95883bdbdb","S":"0x774c2fd6352034ce86e40decae18b75496114b3f720663862132e0ea8b6e2248"}],"feeRatio":"0xa","from":"0x4093593f1b4b24d11315ea0f0182c780ff51d4e7","gas":"0x9c40","gasPrice":"0x5d21dba00","gasUsed":"0x8ca0","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","nonce":"0x3f","senderTxHash":"0xb2dcaccc886f52724d47231b480e2f0e3261491d6eb4282299be014a6d711573","signatures":[{"V":"0x4055","R":"0xbb60e52d6b4f199b173557f5a81cbb04921950d6dcaa873f8bef6bb4fa22f835","S":"0x2c5df20496d1d27f7c65d964e8350084b084e2c202281f4dcf094d02d67c70c2"}],"status":"0x1","to":"0x2a324b72ac10c56a5155ddc75d3465242dc5b866","transactionHash":"0x0d0cc98bb9460a70cb649c221e869d186b5b06048be750bc5a997c96d6d2071d","transactionIndex":"0x0","type":"TxTypeFeeDelegatedValueTransferWithRatio","typeInt":10,"value":"0xe35fa931a0000"}} -------------------------------------------------------------------------------- /klaytnetl/jobs/exporters/tokens_item_exporter.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | from blockchainetl.jobs.exporters.composite_item_exporter import CompositeItemExporter 26 | 27 | FIELDS_TO_EXPORT = [ 28 | "address", 29 | "symbol", 30 | "name", 31 | "decimals", 32 | "total_supply", 33 | "block_number", 34 | ] 35 | 36 | 37 | def tokens_item_exporter(tokens_output): 38 | return CompositeItemExporter( 39 | filename_mapping={"token": tokens_output}, 40 | field_mapping={"token": FIELDS_TO_EXPORT}, 41 | ) 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Klaytn ETL 2 | 3 | Klaytn ETL lets you convert Klaytn blockchain data into convenient formats like JSONs, CSVs and relational databases. 4 | This is a fork of [Ethereum ETL](https://github.com/blockchain-etl/ethereum-etl). 5 | 6 | [Full documentation available here](http://klaytn-etl.readthedocs.io/). 7 | 8 | ***Notice: Klaytn ETL is still on the beta version. However, CLIs are all functional.*** 9 | 10 | ## Quickstart 11 | Install Klaytn ETL: 12 | 13 | ```bash 14 | pip3 install klaytn-etl-cli 15 | ``` 16 | 17 | Export blocks and transactions 18 | 19 | ```bash 20 | > klaytnetl export_blocks_and_transactions --start-block 0 --end-block 5000 \ 21 | --blocks-output blocks.json --transactions-output transactions.json 22 | ``` 23 | 24 | Export ERC20 and ERC721 transfers 25 | 26 | ```bash 27 | > klaytnetl export_token_transfers --start-block 0 --end-block 5000 \ 28 | --output token_transfers.json 29 | ``` 30 | 31 | Export traces 32 | 33 | ```bash 34 | > klaytnetl export_traces --start-block 0 --end-block 5000 \ 35 | --output traces.json 36 | ``` 37 | 38 | Find other commands [here](klaytnetl/cli/__init__.py). 39 | 40 | For the latest version, check out the repo and call 41 | ```bash 42 | > pip3 install -e . 43 | > python3 klaytnetl.py 44 | ``` 45 | 46 | ### Running in Docker 47 | 48 | 1. Install Docker https://docs.docker.com/install/ 49 | 50 | 2. Build a docker image 51 | ```bash 52 | > docker build -t klaytn-etl:latest . 53 | > docker image ls 54 | ``` 55 | 56 | 3. Run a container out of the image 57 | ```bash 58 | > docker run -v $HOME/output:/klaytn-etl/output klaytn-etl:latest export_all -s 0 -e 5499999 -b 100000 59 | > docker run -v $HOME/output:/klaytn-etl/output klaytn-etl:latest export_all -s 2018-01-01 -e 2018-01-01 60 | ``` 61 | -------------------------------------------------------------------------------- /tests/resources/test_export_receipts_job/receipts/web3_response.klay_getTransactionReceipt_0x50a7db8f2ac7564dd6ed0841728c5c43882b136c0130fa4b32989f5403d5e3e7.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":{"blockHash":"0xa60f7854179fb30a5b86174c37e5aeeb69e8b7dadc4194740f7dfd3f1b59d129","blockNumber":"0x4f5d4e1","contractAddress":null,"effectiveGasPrice":"0x5d21dba00","feePayer":"0x10f0eaa52a474f0605afbb8c17bb9189e6ed006c","feePayerSignatures":[{"V":"0x4055","R":"0x88cebfbd16dcd90ffe798fb18c216f4492db4842cec8f9bd477b9601faa3300f","S":"0x66b77280ca5a738dfb71df2ca9583603991efd90c7230874e10532e115c1a8a0"}],"feeRatio":"0x63","from":"0x6aff04024c94d9630dd162193e65933b3a384333","gas":"0xc350","gasPrice":"0x5d21dba00","gasUsed":"0x8e30","input":"0x6d656d6f","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","nonce":"0x1b7464","senderTxHash":"0x451bd829dc58227c7da7c5cd5b6d1d7691b4fe0b958679d732e68ab972640674","signatures":[{"V":"0x4056","R":"0xbc4b754af801049ef3e26fed7feecbc17bcc0e2473f990e83424c795a2988cb7","S":"0x7bbe4599733abad2cc41773d5aaa018da41a828c68f1db9ea529cfc95809e7d7"}],"status":"0x1","to":"0x911fc48da3f6e38d195c6bbfb4fda31bd9b313c5","transactionHash":"0x50a7db8f2ac7564dd6ed0841728c5c43882b136c0130fa4b32989f5403d5e3e7","transactionIndex":"0x384","type":"TxTypeFeeDelegatedValueTransferMemoWithRatio","typeInt":18,"value":"0x1b87506a23714b000000"}} -------------------------------------------------------------------------------- /klaytnetl/erc165_abi.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | import json 26 | 27 | ERC165_ABI = json.loads( 28 | """ 29 | [ 30 | { 31 | "inputs": [ 32 | { 33 | "internalType": "bytes4", 34 | "name": "interfaceId", 35 | "type": "bytes4" 36 | } 37 | ], 38 | "name": "supportsInterface", 39 | "outputs": [ 40 | { 41 | "internalType": "bool", 42 | "name": "", 43 | "type": "bool" 44 | } 45 | ], 46 | "stateMutability": "view", 47 | "type": "function" 48 | } 49 | ] 50 | """ 51 | ) 52 | -------------------------------------------------------------------------------- /tests/klaytnetl/job/mock_ipfs_client.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | import json as JSON 26 | 27 | 28 | class MockIpfsClient: 29 | def __init__(self, read_resource): 30 | self.read_resource = read_resource 31 | 32 | def _get(self, path, json): 33 | ipfs_path = "ipfs_" + path.replace("/", "_") 34 | data = self.read_resource(ipfs_path) 35 | return JSON.loads(data) if json else data 36 | 37 | def get(self, path): 38 | return self._get(path, False) 39 | 40 | def get_json(self, path): 41 | return self._get(path, True) 42 | -------------------------------------------------------------------------------- /blockchainetl/csv_utils.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | 24 | # https://stackoverflow.com/questions/15063936/csv-error-field-larger-than-field-limit-131072 25 | 26 | import sys 27 | import csv 28 | 29 | 30 | def set_max_field_size_limit(): 31 | max_int = sys.maxsize 32 | decrement = True 33 | while decrement: 34 | # decrease the maxInt value by factor 10 35 | # as long as the OverflowError occurs. 36 | 37 | decrement = False 38 | try: 39 | csv.field_size_limit(max_int) 40 | except OverflowError: 41 | max_int = int(max_int / 10) 42 | decrement = True 43 | -------------------------------------------------------------------------------- /klaytnetl/jobs/exporters/contracts_item_exporter.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | from blockchainetl.jobs.exporters.composite_item_exporter import CompositeItemExporter 26 | 27 | FIELDS_TO_EXPORT = [ 28 | "address", 29 | "bytecode", 30 | "function_sighashes", 31 | "is_erc20", 32 | "is_erc721", 33 | "is_erc1155", 34 | "block_number", 35 | ] 36 | 37 | 38 | def contracts_item_exporter(contracts_output): 39 | return CompositeItemExporter( 40 | filename_mapping={"contract": contracts_output}, 41 | field_mapping={"contract": FIELDS_TO_EXPORT}, 42 | ) 43 | -------------------------------------------------------------------------------- /klaytnetl/thread_local_proxy.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | import threading 26 | 27 | 28 | class ThreadLocalProxy: 29 | def __init__(self, delegate_factory): 30 | self._thread_local = threading.local() 31 | self._delegate_factory = delegate_factory 32 | 33 | def __getattr__(self, name): 34 | return getattr(self._get_thread_local_delegate(), name) 35 | 36 | def _get_thread_local_delegate(self): 37 | if getattr(self._thread_local, "_delegate", None) is None: 38 | self._thread_local._delegate = self._delegate_factory() 39 | return self._thread_local._delegate 40 | -------------------------------------------------------------------------------- /klaytnetl/csv_utils.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | # https://stackoverflow.com/questions/15063936/csv-error-field-larger-than-field-limit-131072 26 | 27 | import sys 28 | import csv 29 | 30 | 31 | def set_max_field_size_limit(): 32 | max_int = sys.maxsize 33 | decrement = True 34 | while decrement: 35 | # decrease the maxInt value by factor 10 36 | # as long as the OverflowError occurs. 37 | 38 | decrement = False 39 | try: 40 | csv.field_size_limit(max_int) 41 | except OverflowError: 42 | max_int = int(max_int / 10) 43 | decrement = True 44 | -------------------------------------------------------------------------------- /klaytnetl/jobs/exporters/token_transfers_item_exporter.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | from blockchainetl.jobs.exporters.composite_item_exporter import CompositeItemExporter 26 | 27 | FIELDS_TO_EXPORT = [ 28 | "token_address", 29 | "from_address", 30 | "to_address", 31 | "value", 32 | "transaction_hash", 33 | "log_index", 34 | "block_number", 35 | ] 36 | 37 | 38 | def token_transfers_item_exporter(token_transfer_output): 39 | return CompositeItemExporter( 40 | filename_mapping={"token_transfer": token_transfer_output}, 41 | field_mapping={"token_transfer": FIELDS_TO_EXPORT}, 42 | ) 43 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from setuptools import setup, find_packages 4 | 5 | 6 | def read(fname): 7 | return open(os.path.join(os.path.dirname(__file__), fname)).read() 8 | 9 | def version(): 10 | try: 11 | return read('VERSION').strip().lstrip('v') 12 | except: 13 | return "0.0.0.dev0" 14 | 15 | with open('requirements.txt') as f: 16 | requirements = f.read().splitlines() 17 | 18 | long_description = read('README.md') if os.path.isfile("README.md") else "" 19 | 20 | setup( 21 | name='klaytn-etl-cli', 22 | version=version(), 23 | author='Yongchan Hong', 24 | author_email='chan.hong@krustuniverse.com', 25 | description='Tools for exporting Klaytn blockchain data to JSON', 26 | long_description=long_description, 27 | long_description_content_type='text/markdown', 28 | url='https://github.com/klaytn/klaytn-etl', 29 | packages=find_packages(exclude=['schemas', 'tests']), 30 | classifiers=[ 31 | 'Development Status :: 4 - Beta', 32 | 'Intended Audience :: Developers', 33 | 'License :: OSI Approved :: MIT License', 34 | 'Programming Language :: Python :: 3', 35 | 'Programming Language :: Python :: 3.7', 36 | 'Programming Language :: Python :: 3.8', 37 | 'Programming Language :: Python :: 3.9', 38 | ], 39 | keywords=['klaytn', 'etl', 'batch'], 40 | python_requires='>=3.7.2,<4', 41 | install_requires=requirements, 42 | extras_require={ 43 | 'dev': [ 44 | 'pytest~=4.3.0' 45 | ] 46 | }, 47 | entry_points={ 48 | 'console_scripts': [ 49 | 'klaytnetl=klaytnetl.cli:cli', 50 | ], 51 | }, 52 | project_urls={ 53 | 'Bug Reports': 'https://github.com/klaytn/klaytn-etl/issues', 54 | 'Source': 'https://github.com/klaytn/klaytn-etl/tree/klaytn-etl', 55 | }, 56 | ) 57 | -------------------------------------------------------------------------------- /blockchainetl/jobs/exporters/converters/unix_timestamp_item_converter.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2020 Evgeny Medvedev, evge.medvedev@gmail.com 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | from datetime import datetime 24 | 25 | from blockchainetl.jobs.exporters.converters.simple_item_converter import SimpleItemConverter 26 | 27 | 28 | class UnixTimestampItemConverter(SimpleItemConverter): 29 | 30 | def convert_field(self, key, value): 31 | if key is not None and key.endswith('timestamp'): 32 | return to_timestamp(value) 33 | else: 34 | return value 35 | 36 | 37 | def to_timestamp(value): 38 | if isinstance(value, int): 39 | return datetime.utcfromtimestamp(value).strftime('%Y-%m-%d %H:%M:%S') 40 | else: 41 | return value 42 | -------------------------------------------------------------------------------- /blockchainetl/jobs/exporters/in_memory_item_exporter.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | 24 | class InMemoryItemExporter: 25 | def __init__(self, item_types): 26 | self.item_types = item_types 27 | self.items = {} 28 | 29 | def open(self): 30 | for item_type in self.item_types: 31 | self.items[item_type] = [] 32 | 33 | def export_item(self, item): 34 | item_type = item.get('type', None) 35 | if item_type is None: 36 | raise ValueError('type key is not found in item {}'.format(repr(item))) 37 | 38 | self.items[item_type].append(item) 39 | 40 | def close(self): 41 | pass 42 | 43 | def get_items(self, item_type): 44 | return self.items[item_type] 45 | -------------------------------------------------------------------------------- /blockchainetl/jobs/exporters/kafka_exporter.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import json 3 | import logging 4 | 5 | from kafka import KafkaProducer 6 | 7 | from blockchainetl.jobs.exporters.converters.composite_item_converter import CompositeItemConverter 8 | 9 | 10 | class KafkaItemExporter: 11 | 12 | def __init__(self, output, item_type_to_topic_mapping, converters=()): 13 | self.item_type_to_topic_mapping = item_type_to_topic_mapping 14 | self.converter = CompositeItemConverter(converters) 15 | self.connection_url = self.get_connection_url(output) 16 | print(self.connection_url) 17 | self.producer = KafkaProducer(bootstrap_servers=self.connection_url) 18 | 19 | def get_connection_url(self, output): 20 | try: 21 | return output.split('/')[1] 22 | except KeyError: 23 | raise Exception('Invalid kafka output param, It should be in format of "kafka/127.0.0.1:9092"') 24 | 25 | def open(self): 26 | pass 27 | 28 | def export_items(self, items): 29 | for item in items: 30 | self.export_item(item) 31 | 32 | def export_item(self, item): 33 | item_type = item.get('type') 34 | if item_type is not None and item_type in self.item_type_to_topic_mapping: 35 | data = json.dumps(item).encode('utf-8') 36 | print(data) 37 | return self.producer.send(self.item_type_to_topic_mapping[item_type], value=data) 38 | else: 39 | logging.warning('Topic for item type "{}" is not configured.'.format(item_type)) 40 | 41 | def convert_items(self, items): 42 | for item in items: 43 | yield self.converter.convert_item(item) 44 | 45 | def close(self): 46 | pass 47 | 48 | 49 | def group_by_item_type(items): 50 | result = collections.defaultdict(list) 51 | for item in items: 52 | result[item.get('type')].append(item) 53 | 54 | return result 55 | -------------------------------------------------------------------------------- /klaytnetl/jobs/exporters/traces_item_exporter.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | from blockchainetl.jobs.exporters.composite_item_exporter import CompositeItemExporter 26 | 27 | FIELDS_TO_EXPORT = [ 28 | "block_number", 29 | "transaction_hash", 30 | "transaction_index", 31 | "from_address", 32 | "to_address", 33 | "value", 34 | "input", 35 | "output", 36 | "trace_type", 37 | "call_type", 38 | "reward_type", 39 | "gas", 40 | "gas_used", 41 | "subtraces", 42 | "trace_address", 43 | "error", 44 | "status", 45 | "trace_id", 46 | ] 47 | 48 | 49 | def traces_item_exporter(traces_output): 50 | return CompositeItemExporter( 51 | filename_mapping={"trace": traces_output}, 52 | field_mapping={"trace": FIELDS_TO_EXPORT}, 53 | ) 54 | -------------------------------------------------------------------------------- /klaytnetl/cli/extract_field.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | import click 26 | 27 | from klaytnetl import misc_utils 28 | 29 | 30 | @click.command(context_settings=dict(help_option_names=["-h", "--help"])) 31 | @click.option( 32 | "-i", 33 | "--input", 34 | default="-", 35 | type=str, 36 | help="The input file. If not specified stdin is used.", 37 | ) 38 | @click.option( 39 | "-o", 40 | "--output", 41 | default="-", 42 | type=str, 43 | help="The output file. If not specified stdout is used.", 44 | ) 45 | @click.option( 46 | "-f", "--field", required=True, type=str, help="The field name to extract." 47 | ) 48 | def extract_field(input, output, field): 49 | """Extracts field from given CSV or JSON newline-delimited file.""" 50 | misc_utils.extract_field(input, output, field) 51 | -------------------------------------------------------------------------------- /tests/klaytnetl/job/mock_batch_web3_provider.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | import json 26 | 27 | from tests.klaytnetl.job.mock_web3_provider import MockWeb3Provider, build_file_name 28 | 29 | 30 | class MockBatchWeb3Provider(MockWeb3Provider): 31 | def __init__(self, read_resource): 32 | super().__init__(read_resource) 33 | self.read_resource = read_resource 34 | 35 | def make_batch_request(self, text): 36 | batch = json.loads(text) 37 | web3_response = [] 38 | for req in batch: 39 | method = req["method"] 40 | params = req["params"] 41 | file_name = build_file_name(method, params) 42 | file_content = self.read_resource(file_name) 43 | web3_response.append(json.loads(file_content)) 44 | return web3_response 45 | -------------------------------------------------------------------------------- /blockchainetl/jobs/exporters/converters/simple_item_converter.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | # MIT License 24 | # 25 | # 26 | # Permission is hereby granted, free of charge, to any person obtaining a copy 27 | # of this software and associated documentation files (the "Software"), to deal 28 | # in the Software without restriction, including without limitation the rights 29 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 30 | # copies of the Software, and to permit persons to whom the Software is 31 | # furnished to do so, subject to the following conditions: 32 | 33 | 34 | class SimpleItemConverter: 35 | 36 | def convert_item(self, item): 37 | return { 38 | key: self.convert_field(key, value) for key, value in item.items() 39 | } 40 | 41 | def convert_field(self, key, value): 42 | return value 43 | -------------------------------------------------------------------------------- /klaytnetl/cli/get_keccak_hash.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | import click 26 | 27 | from eth_utils import keccak 28 | 29 | from blockchainetl.file_utils import smart_open 30 | 31 | 32 | @click.command(context_settings=dict(help_option_names=["-h", "--help"])) 33 | @click.option( 34 | "-i", 35 | "--input-string", 36 | default="Transfer(address,address,uint256)", 37 | type=str, 38 | help="String to hash, e.g. Transfer(address,address,uint256)", 39 | ) 40 | @click.option( 41 | "-o", 42 | "--output", 43 | default="-", 44 | type=str, 45 | help="The output file. If not specified stdout is used.", 46 | ) 47 | def get_keccak_hash(input_string, output): 48 | """Outputs 32-byte Keccak hash of given string.""" 49 | hash = keccak(text=input_string) 50 | 51 | with smart_open(output, "w") as output_file: 52 | output_file.write("0x{}\n".format(hash.hex())) 53 | -------------------------------------------------------------------------------- /klaytnetl/mixin/enrichable_mixin.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | class EnrichableMixin(object): 26 | """ 27 | Enrichable mixin 28 | === 29 | The object inheriting this mixin has property `enrich`, which determines an enrichness 30 | of a result item. The `enrich` property has two rules: 31 | 32 | 1. For every enrichable object with `enrich == True`, it only can register enrichable 33 | objects as a child. For all violation cases, it ignores the registration itself. 34 | 2. Enrichness must be inherited to children. For dismatches, it overrides the child's 35 | property. 36 | """ 37 | 38 | def __init__(self, enrich: bool = False, **kwargs): 39 | self.enrich = enrich 40 | 41 | @property 42 | def enrich(self) -> bool: 43 | return self._enrich 44 | 45 | @enrich.setter 46 | def enrich(self, enrich: bool): 47 | self._enrich = enrich 48 | -------------------------------------------------------------------------------- /blockchainetl/jobs/exporters/converters/composite_item_converter.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | # MIT License 24 | # 25 | # 26 | # Permission is hereby granted, free of charge, to any person obtaining a copy 27 | # of this software and associated documentation files (the "Software"), to deal 28 | # in the Software without restriction, including without limitation the rights 29 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 30 | # copies of the Software, and to permit persons to whom the Software is 31 | # furnished to do so, subject to the following conditions: 32 | 33 | 34 | class CompositeItemConverter: 35 | 36 | def __init__(self, converters=()): 37 | self.converters = converters 38 | 39 | def convert_item(self, item): 40 | if self.converters is None: 41 | return item 42 | 43 | for converter in self.converters: 44 | item = converter.convert_item(item) 45 | return item 46 | -------------------------------------------------------------------------------- /klaytnetl/providers/rpc.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # 3 | # Copyright (c) 2016 Piper Merriam 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | 24 | from web3 import HTTPProvider 25 | from web3._utils.request import make_post_request 26 | 27 | 28 | # Mostly copied from web3.py/providers/rpc.py. Supports batch requests. 29 | # Will be removed once batch feature is added to web3.py https://github.com/ethereum/web3.py/issues/832 30 | class BatchHTTPProvider(HTTPProvider): 31 | def make_batch_request(self, text): 32 | self.logger.debug( 33 | "Making request HTTP. URI: %s, Request: %s", self.endpoint_uri, text 34 | ) 35 | request_data = text.encode("utf-8") 36 | raw_response = make_post_request( 37 | self.endpoint_uri, request_data, **self.get_request_kwargs() 38 | ) 39 | response = self.decode_rpc_response(raw_response) 40 | self.logger.debug( 41 | "Getting response HTTP. URI: %s, " "Request: %s, Response: %s", 42 | self.endpoint_uri, 43 | text, 44 | response, 45 | ) 46 | return response 47 | -------------------------------------------------------------------------------- /klaytnetl/executors/fail_safe_executor.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | class FailSafeExecutor: 26 | def __init__(self, delegate): 27 | self._delegate = delegate 28 | self._futures = [] 29 | 30 | def submit(self, fn, *args, **kwargs): 31 | self._check_completed_futures() 32 | future = self._delegate.submit(fn, *args, **kwargs) 33 | self._futures.append(future) 34 | 35 | return future 36 | 37 | def shutdown(self): 38 | self._delegate.shutdown(wait=True) 39 | self._check_completed_futures() 40 | assert len(self._futures) == 0 41 | 42 | def _check_completed_futures(self): 43 | """Fail safe in this case means fail fast. TODO: Add retry logic""" 44 | for future in self._futures.copy(): 45 | if future.done(): 46 | # Will throw an exception here if the future failed 47 | future.result() 48 | self._futures.remove(future) 49 | -------------------------------------------------------------------------------- /klaytnetl/cli/filter_items.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | import click 25 | 26 | from klaytnetl import misc_utils 27 | 28 | 29 | @click.command(context_settings=dict(help_option_names=["-h", "--help"])) 30 | @click.option( 31 | "-i", 32 | "--input", 33 | default="-", 34 | type=str, 35 | help="The input file. If not specified stdin is used.", 36 | ) 37 | @click.option( 38 | "-o", 39 | "--output", 40 | default="-", 41 | type=str, 42 | help="The output file. If not specified stdout is used.", 43 | ) 44 | @click.option( 45 | "-p", 46 | "--predicate", 47 | required=True, 48 | type=str, 49 | help="Predicate in Python code e.g. \"item['is_erc20']\".", 50 | ) 51 | def filter_items(input, output, predicate): 52 | """Filters rows in given CSV or JSON newline-delimited file.""" 53 | 54 | def evaluated_predicate(item): 55 | return eval(predicate, globals(), {"item": item}) 56 | 57 | misc_utils.filter_items(input, output, evaluated_predicate) 58 | -------------------------------------------------------------------------------- /klaytnetl/providers/auto.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | from urllib.parse import urlparse 26 | 27 | from web3 import IPCProvider, HTTPProvider 28 | 29 | from klaytnetl.providers.ipc import BatchIPCProvider 30 | from klaytnetl.providers.rpc import BatchHTTPProvider 31 | 32 | DEFAULT_TIMEOUT = 60 33 | 34 | 35 | def get_provider_from_uri(uri_string, timeout=DEFAULT_TIMEOUT, batch=False): 36 | uri = urlparse(uri_string) 37 | if uri.scheme == "file": 38 | if batch: 39 | return BatchIPCProvider(uri.path, timeout=timeout) 40 | else: 41 | return IPCProvider(uri.path, timeout=timeout) 42 | elif uri.scheme == "http" or uri.scheme == "https": 43 | request_kwargs = {"timeout": timeout} 44 | if batch: 45 | return BatchHTTPProvider(uri_string, request_kwargs=request_kwargs) 46 | else: 47 | return HTTPProvider(uri_string, request_kwargs=request_kwargs) 48 | else: 49 | raise ValueError("Unknown uri scheme {}".format(uri_string)) 50 | -------------------------------------------------------------------------------- /tests/resources/test_export_receipts_job/receipts/web3_response.klay_getTransactionReceipt_0xa8d369dd8aac128b08451a2c08ceda10d9735a8e53c8b3a01aa19df51eda5c79.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":{"blockHash":"0x0e7f8de7731a228886bdb9273bb2f3ea3a9963a2ae49f4a1c73364fb23de0712","blockNumber":"0x5a0e433","codeFormat":"0x0","contractAddress":"0x8b70602c9d9e547c908ba7ef93a157dc8fb286b6","effectiveGasPrice":"0x3a35294400","feePayer":"0x054400f782719b0ea102f52ed21e575f96ed1854","feePayerSignatures":[{"V":"0x4055","R":"0xcf446f74bad22f0425f8e9e8ee9fa24a23ebabcc1bad429b250e6f82fc323004","S":"0x7c755302995c79322e68b7f3b75a61b97170be4e518c17ef603aae53f5a8d755"}],"feeRatio":"0xa","from":"0xa75c9b0a77e242029363808a94ce3f8e1f9d6ea8","gas":"0x2dc6c0","gasPrice":"0x3a35294400","gasUsed":"0x1fe21","humanReadable":false,"input":"0x608060405234801561001057600080fd5b5060c68061001f6000396000f3fe6080604052348015600f57600080fd5b506004361060325760003560e01c80632e64cec11460375780636057361d146053575b600080fd5b603d607e565b6040518082815260200191505060405180910390f35b607c60048036036020811015606757600080fd5b81019080803590602001909291905050506087565b005b60008054905090565b806000819055505056fea265627a7a72315820e79fa321aeedd1f8b6151a9373d5ac76a9d6f82f291abb4ab31a747548ef8e3164736f6c63430005110032","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","nonce":"0xdf","senderTxHash":"0x03910bd0fee29ccacf6bae2b30f5849d1363ac5ce4675f6d077fea055c3f71bc","signatures":[{"V":"0x4056","R":"0xbfb2c4c3c4651d4dfcd1e8d4413aaf63932587044a5b4f8be9247ae129c70844","S":"0x4fc47a66fb676d598169dab6a7ff88eb5193c005ddcfd3e45620037c0d6f3215"}],"status":"0x1","to":null,"transactionHash":"0xa8d369dd8aac128b08451a2c08ceda10d9735a8e53c8b3a01aa19df51eda5c79","transactionIndex":"0x0","type":"TxTypeFeeDelegatedSmartContractDeployWithRatio","typeInt":42,"value":"0x0"}} -------------------------------------------------------------------------------- /tests/resources/test_export_receipts_job/receipts/web3_response.klay_getTransactionReceipt_0xbc22a4cd65003337d6bc70297df4401cab342e710f67769dc13999df92b34306.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":{"blockHash":"0xbf015b52e8aa61c19980a5f85ca868cf8e9b9965d8c78093c34ead4f17dd14d7","blockNumber":"0x5d97094","contractAddress":null,"effectiveGasPrice":"0x3a35294400","from":"0xd5e60614cfa6f6344fac259c3a75686fca8925f0","gas":"0x186a0","gasPrice":"0x3a35294400","gasUsed":"0x9858","input":"0xf8b280b8aff8ada03f3806140fc29333a69fe3c6f79047c76af5b84bb6889ddd554a54390f6642b3a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470a081a168d47641d88364191036df1b6752bf04e5c229dbd1fbc87b7a818be99823a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470a0a3d6247ca862ccf44dddfd510c45ade4a371993d9cebad4661f701e8b16b998583352c8082025857","inputJSON":{"blockHash":"0x3f3806140fc29333a69fe3c6f79047c76af5b84bb6889ddd554a54390f6642b3","transactionsRoot":"0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470","parentHash":"0x81a168d47641d88364191036df1b6752bf04e5c229dbd1fbc87b7a818be99823","receiptsRoot":"0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470","stateRoot":"0xa3d6247ca862ccf44dddfd510c45ade4a371993d9cebad4661f701e8b16b9985","blockNumber":3484800,"blockCount":600,"txCount":87},"logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","nonce":"0x38d4","senderTxHash":"0xbc22a4cd65003337d6bc70297df4401cab342e710f67769dc13999df92b34306","signatures":[{"V":"0x4055","R":"0x8dbcbfbb40a32390f7626fde433474282e788d027e7c04fcc263780a58f4e6ee","S":"0x5c72abdc25b7eb6fb793f191e8266db80bce188d36d36798ad9b206427b1d29"}],"status":"0x1","transactionHash":"0xbc22a4cd65003337d6bc70297df4401cab342e710f67769dc13999df92b34306","transactionIndex":"0x0","type":"TxTypeChainDataAnchoring","typeInt":72}} -------------------------------------------------------------------------------- /blockchainetl/jobs/exporters/converters/int_to_string_item_converter.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | # MIT License 24 | # 25 | # 26 | # Permission is hereby granted, free of charge, to any person obtaining a copy 27 | # of this software and associated documentation files (the "Software"), to deal 28 | # in the Software without restriction, including without limitation the rights 29 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 30 | # copies of the Software, and to permit persons to whom the Software is 31 | # furnished to do so, subject to the following conditions: 32 | 33 | 34 | from blockchainetl.jobs.exporters.converters.simple_item_converter import SimpleItemConverter 35 | 36 | 37 | class IntToStringItemConverter(SimpleItemConverter): 38 | 39 | def __init__(self, keys=None): 40 | self.keys = set(keys) if keys else None 41 | 42 | def convert_field(self, key, value): 43 | if isinstance(value, int) and (self.keys is None or key in self.keys): 44 | return str(value) 45 | else: 46 | return value 47 | -------------------------------------------------------------------------------- /klaytnetl/jobs/extract_tokens_job.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | from klaytnetl.jobs.export_tokens_job import ExportTokensJob 26 | 27 | 28 | class ExtractTokensJob(ExportTokensJob): 29 | def __init__(self, web3, item_exporter, contracts_iterable, max_workers): 30 | super().__init__(web3, item_exporter, [], max_workers) 31 | self.contracts_iterable = contracts_iterable 32 | 33 | def _export(self): 34 | self.batch_work_executor.execute( 35 | self.contracts_iterable, self._export_tokens_from_contracts 36 | ) 37 | 38 | def _export_tokens_from_contracts(self, contracts): 39 | tokens = [ 40 | contract 41 | for contract in contracts 42 | if contract.get("is_erc20") 43 | or contract.get("is_erc721") 44 | or contract.get("is_erc1155") 45 | ] 46 | 47 | for token in tokens: 48 | self._export_token( 49 | token_address=token["address"], block_number=token["block_number"] 50 | ) 51 | -------------------------------------------------------------------------------- /tests/resources/test_export_receipts_job/receipts/web3_response.klay_getTransactionReceipt_0xb93317dd74dd711b01a48debacb672e6c75bf000ee4e23831d85fce99eef576f.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":{"blockHash":"0xc6dd72c040a01346a3e9a6b823547bdd7e967a03a8f2b7ae59e9d962ceae53e4","blockNumber":"0x5d97306","contractAddress":null,"effectiveGasPrice":"0x3a35294400","from":"0x3654854df24d58c78f04b745dc2d885042230ce6","gas":"0x3e962","gasPrice":"0x3a35294400","gasUsed":"0x17f81","input":"0x23b872dd0000000000000000000000003654854df24d58c78f04b745dc2d885042230ce6000000000000000000000000d42b343ff64f069cd3e57220d4279180e53772520000000000000000000000000000000000000000000000000000000000005582","logs":[{"address":"0x06ce4f100f9e61752d8fb792948f449e69baad1f","topics":["0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef","0x0000000000000000000000003654854df24d58c78f04b745dc2d885042230ce6","0x000000000000000000000000d42b343ff64f069cd3e57220d4279180e5377252","0x0000000000000000000000000000000000000000000000000000000000005582"],"data":"0x","blockNumber":"0x5d97306","transactionHash":"0xb93317dd74dd711b01a48debacb672e6c75bf000ee4e23831d85fce99eef576f","transactionIndex":"0x1","blockHash":"0xc6dd72c040a01346a3e9a6b823547bdd7e967a03a8f2b7ae59e9d962ceae53e4","logIndex":"0x1","removed":false}],"logsBloom":"0x00000000000000000000000000000000000000004000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000200000000000000000000000000000000000000080000000010000000000000000000000000000000400000000000000001080000000000000000000000000000000020000000000000000000000000000000000000000000000000080000000002000000800000000008000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000","nonce":"0x111","senderTxHash":"0xb93317dd74dd711b01a48debacb672e6c75bf000ee4e23831d85fce99eef576f","signatures":[{"V":"0x4055","R":"0x7700d401c3ad0dc030b087712dd1e5c157261dd6a6a875adba5737f6bc2ac0c3","S":"0x1adb7d2fb162aacc7f512251d6506a075817f899681c193ceefcd95d70dc9207"}],"status":"0x1","to":"0x06ce4f100f9e61752d8fb792948f449e69baad1f","transactionHash":"0xb93317dd74dd711b01a48debacb672e6c75bf000ee4e23831d85fce99eef576f","transactionIndex":"0x1","type":"TxTypeSmartContractExecution","typeInt":48,"value":"0x0"}} -------------------------------------------------------------------------------- /tests/klaytnetl/job/mock_web3_provider.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | import json 26 | 27 | from web3 import IPCProvider 28 | 29 | 30 | class MockWeb3Provider(IPCProvider): 31 | def __init__(self, read_resource): 32 | self.read_resource = read_resource 33 | 34 | def make_request(self, method, params): 35 | file_name = build_file_name(method, params) 36 | file_content = self.read_resource(file_name) 37 | return json.loads(file_content) 38 | 39 | 40 | def build_file_name(method, params): 41 | return ( 42 | "web3_response." 43 | + method 44 | + "_" 45 | + "_".join([param_to_str(param) for param in params]) 46 | + ".json" 47 | ) 48 | 49 | 50 | def param_to_str(param): 51 | if isinstance(param, dict): 52 | return "_".join( 53 | [str(key) + "_" + param_to_str(param[key]) for key in sorted(param)] 54 | ) 55 | elif isinstance(param, list): 56 | return "_".join([param_to_str(param_item) for param_item in param]) 57 | else: 58 | return str(param).lower() 59 | -------------------------------------------------------------------------------- /schemas/logs.py: -------------------------------------------------------------------------------- 1 | LOGS_SCHEMA = [ 2 | { 3 | "name": "block_number", 4 | "type": "INT64", 5 | "mode": "NULLABLE", 6 | "description": "Block number corresponding", 7 | }, 8 | { 9 | "name": "block_hash", 10 | "type": "STRING", 11 | "mode": "NULLABLE", 12 | "description": "Hash of the block", 13 | }, 14 | { 15 | "name": "block_timestamp", 16 | "type": "TIMESTAMP", 17 | "mode": "NULLABLE", 18 | "description": "The UTC timestamp for when the block was collated", 19 | }, 20 | { 21 | "name": "block_unix_timestamp", 22 | "type": "FLOAT64", 23 | "mode": "NULLABLE", 24 | "description": "The unix timestamp for when the block was collated", 25 | }, 26 | { 27 | "name": "transaction_hash", 28 | "type": "STRING", 29 | "mode": "NULLABLE", 30 | "description": "Integer of the transactions index position in the block", 31 | }, 32 | { 33 | "name": "transaction_index", 34 | "type": "INT64", 35 | "mode": "NULLABLE", 36 | "description": "Hash of the transactions", 37 | }, 38 | { 39 | "name": "transaction_receipt_status", 40 | "type": "INT64", 41 | "mode": "NULLABLE", 42 | "description": "Either 1 (success) or 0 (failure) (post Byzantium)", 43 | }, 44 | { 45 | "name": "log_index", 46 | "type": "INT64", 47 | "mode": "NULLABLE", 48 | "description": "Integer of the log index position in the block", 49 | }, 50 | { 51 | "name": "address", 52 | "type": "STRING", 53 | "mode": "NULLABLE", 54 | "description": "Address from which this log originated", 55 | }, 56 | { 57 | "name": "data", 58 | "type": "STRING", 59 | "mode": "NULLABLE", 60 | "description": "Contains one or more 32 Bytes non-indexed arguments of the log", 61 | }, 62 | { 63 | "name": "topics", 64 | "type": "STRING", 65 | "mode": "REPEATED", 66 | "description": "Indexed log arguments (0 to 4 32-byte hex strings). (In solidity: The first topic is the hash of the signature of the event (e.g. Deposit(address,bytes32,uint256)), except you declared the event with the anonymous specifier.)", 67 | }, 68 | ] 69 | -------------------------------------------------------------------------------- /blockchainetl/jobs/exporters/converters/int_to_decimal_item_converter.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | # MIT License 24 | # 25 | # 26 | # Permission is hereby granted, free of charge, to any person obtaining a copy 27 | # of this software and associated documentation files (the "Software"), to deal 28 | # in the Software without restriction, including without limitation the rights 29 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 30 | # copies of the Software, and to permit persons to whom the Software is 31 | # furnished to do so, subject to the following conditions: 32 | # 33 | # 34 | 35 | from decimal import Decimal 36 | 37 | from blockchainetl.jobs.exporters.converters.simple_item_converter import SimpleItemConverter 38 | 39 | # Large ints are not handled correctly by pg8000 so we use Decimal instead: 40 | # https://github.com/mfenniak/pg8000/blob/412eace074514ada824e7a102765e37e2cda8eaa/pg8000/core.py#L1703 41 | class IntToDecimalItemConverter(SimpleItemConverter): 42 | 43 | def convert_field(self, key, value): 44 | if isinstance(value, int): 45 | return Decimal(value) 46 | else: 47 | return value 48 | -------------------------------------------------------------------------------- /klaytnetl/cli/extract_csv_column.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | import click 26 | import csv 27 | 28 | from klaytnetl.csv_utils import set_max_field_size_limit 29 | from blockchainetl.file_utils import smart_open 30 | 31 | 32 | @click.command(context_settings=dict(help_option_names=["-h", "--help"])) 33 | @click.option( 34 | "-i", 35 | "--input", 36 | default="-", 37 | type=str, 38 | help="The input file. If not specified stdin is used.", 39 | ) 40 | @click.option( 41 | "-o", 42 | "--output", 43 | default="-", 44 | type=str, 45 | help="The output file. If not specified stdout is used.", 46 | ) 47 | @click.option( 48 | "-c", "--column", required=True, type=str, help="The csv column name to extract." 49 | ) 50 | def extract_csv_column(input, output, column): 51 | """Extracts column from given CSV file. Deprecated - use extract_field.""" 52 | set_max_field_size_limit() 53 | 54 | with smart_open(input, "r") as input_file, smart_open(output, "w") as output_file: 55 | reader = csv.DictReader(input_file) 56 | for row in reader: 57 | output_file.write(row[column] + "\n") 58 | -------------------------------------------------------------------------------- /klaytnetl/cli/s3_sync.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | import os 26 | import logging 27 | import boto3 28 | 29 | 30 | def get_path(tmpdir, path): 31 | if path is None: 32 | return None 33 | elif tmpdir is None: 34 | return os.path.normpath(path) 35 | else: 36 | return os.path.normpath(os.path.join(tmpdir, path)) 37 | 38 | 39 | def sync_to_s3(s3_bucket, tmpdir, outputs, is_single_file): 40 | s3 = boto3.resource("s3") 41 | 42 | for output_path in filter(lambda o: o is not None, outputs): 43 | temp_path = get_path(tmpdir, output_path) 44 | for file in _get_files(temp_path, is_single_file): 45 | out = os.path.join(os.path.normpath(output_path), os.path.basename(file)) 46 | logging.info(f"Transfer {file} --> s3://{s3_bucket}/{out}") 47 | s3.Object(s3_bucket, out).put(Body=open(file, "rb")) 48 | 49 | 50 | def _get_files(path, is_single_file): 51 | import os, glob 52 | 53 | if is_single_file: 54 | path = os.path.normpath(path) 55 | else: 56 | path = os.path.join(os.path.normpath(path), "*") 57 | 58 | return glob.glob(path) 59 | -------------------------------------------------------------------------------- /schemas/token_transfers.py: -------------------------------------------------------------------------------- 1 | TOKEN_TRANSFERS_SCHEMA = [ 2 | { 3 | "name": "token_address", 4 | "type": "STRING", 5 | "mode": "NULLABLE", 6 | "description": "Token address", 7 | }, 8 | { 9 | "name": "from_address", 10 | "type": "STRING", 11 | "mode": "NULLABLE", 12 | "description": "Address of the sender", 13 | }, 14 | { 15 | "name": "to_address", 16 | "type": "STRING", 17 | "mode": "NULLABLE", 18 | "description": "Address of the receiver", 19 | }, 20 | { 21 | "name": "value", 22 | "type": "STRING", 23 | "mode": "NULLABLE", 24 | "description": "Amount of tokens transferred (ERC20) / id of the token transferred (ERC721). Use safe_cast for casting to NUMERIC or FLOAT64", 25 | }, 26 | { 27 | "name": "block_hash", 28 | "type": "STRING", 29 | "mode": "NULLABLE", 30 | "description": "Hash of the block", 31 | }, 32 | { 33 | "name": "block_number", 34 | "type": "INT64", 35 | "mode": "NULLABLE", 36 | "description": "Block number corresponding", 37 | }, 38 | { 39 | "name": "block_timestamp", 40 | "type": "TIMESTAMP", 41 | "mode": "NULLABLE", 42 | "description": "The UTC timestamp for when the block was collated", 43 | }, 44 | { 45 | "name": "block_unix_timestamp", 46 | "type": "FLOAT64", 47 | "mode": "NULLABLE", 48 | "description": "The unix timestamp for when the block was collated", 49 | }, 50 | { 51 | "name": "transaction_hash", 52 | "type": "STRING", 53 | "mode": "NULLABLE", 54 | "description": "Hash of the transactions", 55 | }, 56 | { 57 | "name": "transaction_index", 58 | "type": "INT64", 59 | "mode": "NULLABLE", 60 | "description": "Integer of the transactions index position in the block", 61 | }, 62 | { 63 | "name": "transaction_receipt_status", 64 | "type": "INT64", 65 | "mode": "NULLABLE", 66 | "description": "Either 1 (success) or 0 (failure) (post Byzantium)", 67 | }, 68 | { 69 | "name": "log_index", 70 | "type": "INT64", 71 | "mode": "NULLABLE", 72 | "description": "Integer of the log index position in the block", 73 | }, 74 | ] 75 | -------------------------------------------------------------------------------- /tests/resources/test_export_receipts_job/receipts/web3_response.klay_getTransactionReceipt_0x5b665efa2da5a58da36af0e854734273247b4af3f38892412c07447ddde2a65a.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":{"blockHash":"0x7bcf24bab3f10a501a6bb5e7f2ea5bd9bed58a3b9a57dce32a07136586aeb925","blockNumber":"0x58e3012","contractAddress":null,"effectiveGasPrice":"0x3a35294400","feePayer":"0xcd11d51e7beb882af7c19c9b32a9035f35f03a7c","feePayerSignatures":[{"V":"0x4056","R":"0x87d09e9e9f40042d2aca0483cc061a0abfd94c5bbc8a16eb1e60861c9462442c","S":"0x50d0ad78e9ba203316b32a9abfc30daff4dc6ffc8c81fd101e41741fb1a5c158"}],"feeRatio":"0x50","from":"0x6a37283169e01e2296200ebd40ef3a0cc011f5f8","gas":"0x25f04","gasPrice":"0x3a35294400","gasUsed":"0xa561","input":"0xa9059cbb000000000000000000000000cd11d51e7beb882af7c19c9b32a9035f35f03a7c0000000000000000000000000000000000000000000000000027147114878000","logs":[{"address":"0x5096db80b21ef45230c9e423c373f1fc9c0198dd","topics":["0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef","0x0000000000000000000000006a37283169e01e2296200ebd40ef3a0cc011f5f8","0x000000000000000000000000cd11d51e7beb882af7c19c9b32a9035f35f03a7c"],"data":"0x0000000000000000000000000000000000000000000000000027147114878000","blockNumber":"0x58e3012","transactionHash":"0x5b665efa2da5a58da36af0e854734273247b4af3f38892412c07447ddde2a65a","transactionIndex":"0x2","blockHash":"0x7bcf24bab3f10a501a6bb5e7f2ea5bd9bed58a3b9a57dce32a07136586aeb925","logIndex":"0xd","removed":false}],"logsBloom":"0x00000200000000000000000000000000000000000000000000000000008000000000000000000000000000100000000000000000000000000000000000000000000000000000000020000008000000000400000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000800000008000000000000000000000000000000000000000000000000000001002000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000","nonce":"0x4","senderTxHash":"0x255f5db580b7f81f438fb4e5ffb180fd6ee32d87f2027cee848febb7e53911ad","signatures":[{"V":"0x4055","R":"0xe0e8b5c8334a54b335ac89fc7c9f9b1b7e5d681b9283b0556dc28d0bdd15ff4b","S":"0x68ab38008acedb4317e02e5215768b58b7e6e38375f7cacd54299b00dff7fce3"}],"status":"0x1","to":"0x5096db80b21ef45230c9e423c373f1fc9c0198dd","transactionHash":"0x5b665efa2da5a58da36af0e854734273247b4af3f38892412c07447ddde2a65a","transactionIndex":"0x2","type":"TxTypeFeeDelegatedSmartContractExecutionWithRatio","typeInt":50,"value":"0x0"}} -------------------------------------------------------------------------------- /klaytnetl/executors/bounded_executor.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | from concurrent.futures import ThreadPoolExecutor 26 | from threading import BoundedSemaphore 27 | 28 | 29 | class BoundedExecutor: 30 | """BoundedExecutor behaves as a ThreadPoolExecutor which will block on 31 | calls to submit() once the limit given as "bound" work items are queued for 32 | execution. 33 | :param bound: Integer - the maximum number of items in the work queue 34 | :param max_workers: Integer - the size of the thread pool 35 | """ 36 | 37 | def __init__(self, bound, max_workers): 38 | self._delegate = ThreadPoolExecutor(max_workers=max_workers) 39 | self._semaphore = BoundedSemaphore(bound + max_workers) 40 | 41 | """See concurrent.futures.Executor#submit""" 42 | 43 | def submit(self, fn, *args, **kwargs): 44 | self._semaphore.acquire() 45 | try: 46 | future = self._delegate.submit(fn, *args, **kwargs) 47 | except: 48 | self._semaphore.release() 49 | raise 50 | else: 51 | future.add_done_callback(lambda x: self._semaphore.release()) 52 | return future 53 | 54 | """See concurrent.futures.Executor#shutdown""" 55 | 56 | def shutdown(self, wait=True): 57 | self._delegate.shutdown(wait) 58 | -------------------------------------------------------------------------------- /tests/klaytnetl/job/helpers.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | import os 26 | 27 | from web3 import HTTPProvider 28 | 29 | from klaytnetl.providers.rpc import BatchHTTPProvider 30 | from tests.klaytnetl.job.mock_batch_web3_provider import MockBatchWeb3Provider 31 | from tests.klaytnetl.job.mock_web3_provider import MockWeb3Provider 32 | 33 | 34 | def get_web3_provider(provider_type, read_resource_lambda=None, batch=False): 35 | if provider_type == "mock": 36 | if read_resource_lambda is None: 37 | raise ValueError( 38 | "read_resource_lambda must not be None for provider type mock".format( 39 | provider_type 40 | ) 41 | ) 42 | if batch: 43 | provider = MockBatchWeb3Provider(read_resource_lambda) 44 | else: 45 | provider = MockWeb3Provider(read_resource_lambda) 46 | elif provider_type == "fantrie": 47 | provider_url = os.environ.get( 48 | "PROVIDER_URL", "https://archive-en.node.kaia.io" 49 | ) 50 | if batch: 51 | provider = BatchHTTPProvider(provider_url) 52 | else: 53 | provider = HTTPProvider(provider_url) 54 | else: 55 | raise ValueError("Provider type {} is unexpected".format(provider_type)) 56 | return provider 57 | -------------------------------------------------------------------------------- /tests/helpers.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | import os 26 | import json 27 | 28 | import pytest 29 | 30 | 31 | def sort_json(json_string): 32 | return json.dumps(json.loads(json_string), sort_keys=True) 33 | 34 | 35 | def compare_lines_ignore_order(expected, actual): 36 | expected_lines = expected.splitlines() 37 | actual_lines = actual.splitlines() 38 | assert len(expected_lines) == len(actual_lines) 39 | 40 | try: 41 | expected_lines = [sort_json(line) for line in expected_lines] 42 | actual_lines = [sort_json(line) for line in actual_lines] 43 | except json.decoder.JSONDecodeError: 44 | pass 45 | 46 | for expected_line, actual_line in zip(sorted(expected_lines), sorted(actual_lines)): 47 | assert expected_line == actual_line 48 | 49 | 50 | def read_file(path): 51 | if not os.path.exists(path): 52 | return "" 53 | with open(path) as file: 54 | return file.read() 55 | 56 | 57 | run_slow_tests_variable = os.environ.get("KLAYTN_ETL_RUN_SLOW_TESTS", "False") 58 | run_slow_tests = run_slow_tests_variable.lower() in ["1", "true", "yes"] 59 | 60 | 61 | def skip_if_slow_tests_disabled(data): 62 | return pytest.param( 63 | *data, 64 | marks=pytest.mark.skipif( 65 | not run_slow_tests, reason="Skipping slow running tests" 66 | ) 67 | ) 68 | -------------------------------------------------------------------------------- /tests/klaytnetl/job/test_extract_token_transfers_job.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | import csv 26 | import io 27 | 28 | import pytest 29 | 30 | import tests.resources 31 | from klaytnetl.jobs.exporters.token_transfers_item_exporter import ( 32 | token_transfers_item_exporter, 33 | ) 34 | from klaytnetl.jobs.extract_token_transfers_job import ExtractTokenTransfersJob 35 | from tests.helpers import compare_lines_ignore_order, read_file 36 | 37 | RESOURCE_GROUP = "test_extract_token_transfers_job" 38 | 39 | 40 | def read_resource(resource_group, file_name): 41 | return tests.resources.read_resource([RESOURCE_GROUP, resource_group], file_name) 42 | 43 | 44 | @pytest.mark.parametrize("resource_group", ["logs"]) 45 | def test_export_token_transfers_job(tmpdir, resource_group): 46 | output_file = str(tmpdir.join("token_transfers.csv")) 47 | 48 | logs_content = read_resource(resource_group, "logs.csv") 49 | logs_csv_reader = csv.DictReader(io.StringIO(logs_content)) 50 | 51 | job = ExtractTokenTransfersJob( 52 | logs_iterable=logs_csv_reader, 53 | batch_size=2, 54 | item_exporter=token_transfers_item_exporter(output_file), 55 | max_workers=5, 56 | ) 57 | job.run() 58 | 59 | compare_lines_ignore_order( 60 | read_resource(resource_group, "expected_token_transfers.csv"), 61 | read_file(output_file), 62 | ) 63 | -------------------------------------------------------------------------------- /klaytnetl/cli/gcs_sync.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | import os 26 | import glob 27 | import logging 28 | import shutil 29 | 30 | from google.cloud import storage 31 | from google.oauth2 import service_account 32 | 33 | 34 | def get_path(tmpdir, path): 35 | if path is None: 36 | return None 37 | elif tmpdir is None: 38 | return os.path.normpath(path) 39 | else: 40 | return os.path.normpath(os.path.join(tmpdir, path)) 41 | 42 | 43 | def sync_to_gcs(bucket, tmpdir, outputs, is_single_file): 44 | client = storage.Client() 45 | bucket_name = client.get_bucket(bucket) 46 | 47 | for output_path in filter(lambda o: o is not None, outputs): 48 | temp_path = get_path(tmpdir, output_path) 49 | for file in _get_files(temp_path, is_single_file): 50 | out = os.path.join(os.path.normpath(output_path), os.path.basename(file)) 51 | logging.info(f"Transfer {file} --> gcs://{bucket}/{out}") 52 | blob = bucket_name.blob(f'{bucket}/{out}') 53 | blob.upload_from_filename(f'{file}') 54 | shutil.rmtree(temp_path, ignore_errors=True) 55 | 56 | 57 | def _get_files(path, is_single_file): 58 | import os, glob 59 | 60 | if is_single_file: 61 | path = os.path.normpath(path) 62 | else: 63 | path = os.path.join(os.path.normpath(path), "*") 64 | 65 | return glob.glob(path) 66 | -------------------------------------------------------------------------------- /klaytnetl/jobs/exporters/raw_traces_item_exporter.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | from blockchainetl.jobs.exporters.singlefile_item_exporter import SinglefileItemExporter 26 | from blockchainetl.jobs.exporters.multifile_item_exporter import MultifileItemExporter 27 | 28 | FIELDS_TO_EXPORT = [ 29 | "block_number", 30 | "transaction_hash", 31 | "transaction_index", 32 | "from_address", 33 | "to_address", 34 | "value", 35 | "input", 36 | "output", 37 | "trace_type", 38 | "call_type", 39 | "gas", 40 | "gas_used", 41 | "subtraces", 42 | "trace_address", 43 | "error", 44 | "status", 45 | "trace_index", 46 | ] 47 | 48 | 49 | def raw_traces_item_exporter(traces_output, **kwargs): 50 | maxlines = kwargs.get("file_maxlines", None) 51 | 52 | if maxlines is None or maxlines <= 0: 53 | return SinglefileItemExporter( 54 | filename_mapping={ 55 | "trace": traces_output, 56 | }, 57 | field_mapping={ 58 | "trace": FIELDS_TO_EXPORT, 59 | }, 60 | **kwargs 61 | ) 62 | else: 63 | return MultifileItemExporter( 64 | dirname_mapping={ 65 | "trace": traces_output, 66 | }, 67 | field_mapping={ 68 | "trace": FIELDS_TO_EXPORT, 69 | }, 70 | **kwargs 71 | ) 72 | -------------------------------------------------------------------------------- /blockchainetl/jobs/exporters/converters/list_field_item_converter.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | # MIT License 24 | # 25 | # 26 | # Permission is hereby granted, free of charge, to any person obtaining a copy 27 | # of this software and associated documentation files (the "Software"), to deal 28 | # in the Software without restriction, including without limitation the rights 29 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 30 | # copies of the Software, and to permit persons to whom the Software is 31 | # furnished to do so, subject to the following conditions: 32 | 33 | 34 | class ListFieldItemConverter: 35 | 36 | def __init__(self, field, new_field_prefix, fill=0, fill_with=None): 37 | self.field = field 38 | self.new_field_prefix = new_field_prefix 39 | self.fill = fill 40 | self.fill_with = fill_with 41 | 42 | def convert_item(self, item): 43 | if not item: 44 | return item 45 | 46 | lst = item.get(self.field) 47 | result = item 48 | if lst is not None and isinstance(lst, list): 49 | result = item.copy() 50 | del result[self.field] 51 | for lst_item_index, lst_item in enumerate(lst): 52 | result[self.new_field_prefix + str(lst_item_index)] = lst_item 53 | if len(lst) < self.fill: 54 | for i in range(len(lst), self.fill): 55 | result[self.new_field_prefix + str(i)] = self.fill_with 56 | return result 57 | -------------------------------------------------------------------------------- /klaytnetl/jobs/exporters/enrich_traces_item_exporter.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | from blockchainetl.jobs.exporters.singlefile_item_exporter import SinglefileItemExporter 26 | from blockchainetl.jobs.exporters.multifile_item_exporter import MultifileItemExporter 27 | 28 | FIELDS_TO_EXPORT = [ 29 | "block_number", 30 | "block_hash", 31 | "block_timestamp", 32 | "block_unix_timestamp", 33 | "transaction_hash", 34 | "transaction_index", 35 | "transaction_receipt_status", 36 | "from_address", 37 | "to_address", 38 | "value", 39 | "input", 40 | "output", 41 | "trace_type", 42 | "call_type", 43 | "gas", 44 | "gas_used", 45 | "subtraces", 46 | "trace_address", 47 | "error", 48 | "status", 49 | "trace_index", 50 | ] 51 | 52 | 53 | def enrich_traces_item_exporter(traces_output, **kwargs): 54 | maxlines = kwargs.get("file_maxlines", None) 55 | 56 | if maxlines is None or maxlines <= 0: 57 | return SinglefileItemExporter( 58 | filename_mapping={ 59 | "trace": traces_output, 60 | }, 61 | field_mapping={ 62 | "trace": FIELDS_TO_EXPORT, 63 | }, 64 | **kwargs 65 | ) 66 | else: 67 | return MultifileItemExporter( 68 | dirname_mapping={ 69 | "trace": traces_output, 70 | }, 71 | field_mapping={ 72 | "trace": FIELDS_TO_EXPORT, 73 | }, 74 | **kwargs 75 | ) 76 | -------------------------------------------------------------------------------- /klaytnetl/jobs/exporters/receipts_and_logs_item_exporter.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | from blockchainetl.jobs.exporters.composite_item_exporter import CompositeItemExporter 26 | 27 | RECEIPT_FIELDS_TO_EXPORT = [ 28 | "transaction_hash", 29 | "transaction_index", 30 | "block_hash", 31 | "block_number", 32 | "gas", 33 | "gas_price", 34 | "gas_used", 35 | "effective_gas_price", 36 | "contract_address", 37 | "logs_bloom", 38 | "nonce", 39 | "fee_payer", 40 | "fee_payer_signatures", 41 | "fee_ratio", 42 | "code_format", 43 | "human_readable", 44 | "tx_error", 45 | "key", 46 | "input_data", 47 | "from_address", 48 | "to_address", 49 | "type_name", 50 | "type_int", 51 | "sender_tx_hash", 52 | "signatures", 53 | "status", 54 | "value", 55 | "input_json", 56 | "access_list", 57 | "chain_id", 58 | "max_priority_fee_per_gas", 59 | "max_fee_per_gas", 60 | ] 61 | 62 | LOG_FIELDS_TO_EXPORT = [ 63 | "log_index", 64 | "transaction_hash", 65 | "transaction_index", 66 | "block_hash", 67 | "block_number", 68 | "address", 69 | "data", 70 | "topics", 71 | "removed", 72 | ] 73 | 74 | 75 | def receipts_and_logs_item_exporter(receipts_output=None, logs_output=None): 76 | return CompositeItemExporter( 77 | filename_mapping={"receipt": receipts_output, "log": logs_output}, 78 | field_mapping={ 79 | "receipt": RECEIPT_FIELDS_TO_EXPORT, 80 | "log": LOG_FIELDS_TO_EXPORT, 81 | }, 82 | ) 83 | -------------------------------------------------------------------------------- /klaytnetl/service/trace_block_service.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | import logging 26 | 27 | 28 | from typing import Iterable 29 | 30 | from klaytnetl.utils import is_empty_trace_result 31 | 32 | logger = logging.getLogger("trace_block_service") 33 | 34 | 35 | def iterate_transaction_traces( 36 | transaction_traces: list, block_transactions: list = None 37 | ) -> Iterable: 38 | if block_transactions is not None: 39 | assert len(transaction_traces) == len( 40 | block_transactions 41 | ), "ValueError: transaction_traces and block_transactions must have same cardinality." 42 | else: 43 | logger.warn( 44 | "ValueWarn: A block_transactions field was not provided. The result will be with not enough information." 45 | ) 46 | 47 | for idx, trace in enumerate(transaction_traces): 48 | if trace is None or is_empty_trace_result(trace): 49 | if block_transactions is not None: 50 | yield block_transactions[idx] 51 | else: 52 | yield None 53 | else: 54 | if block_transactions is not None: 55 | trace["transactionHash"] = block_transactions[idx].get( 56 | "transactionHash" 57 | ) 58 | trace["transactionIndex"] = block_transactions[idx].get( 59 | "transactionIndex" 60 | ) 61 | trace["transactionReceiptStatus"] = block_transactions[idx].get( 62 | "status" 63 | ) 64 | yield trace 65 | -------------------------------------------------------------------------------- /tests/klaytnetl/test_progress_logger.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | from klaytnetl.progress_logger import ProgressLogger 26 | 27 | 28 | def test_progress_logger(): 29 | logger_mock = LoggerMock() 30 | progress_logger = ProgressLogger(logger=logger_mock, log_item_step=1000) 31 | 32 | progress_logger.start() 33 | [progress_logger.track(100) for _ in range(100)] 34 | progress_logger.finish() 35 | 36 | assert len(logger_mock.logs) == 12 37 | assert logger_mock.logs[0] == "Started work." 38 | assert logger_mock.logs[1] == "1000 items processed." 39 | assert logger_mock.logs[11].startswith( 40 | "Finished work. Total items processed: 10000. Took " 41 | ) 42 | 43 | 44 | def test_progress_logger_with_total_items(): 45 | logger_mock = LoggerMock() 46 | progress_logger = ProgressLogger(logger=logger_mock, log_percentage_step=5) 47 | 48 | progress_logger.start(total_items=1234) 49 | [progress_logger.track(99) for _ in range(100)] 50 | progress_logger.finish() 51 | 52 | assert len(logger_mock.logs) == 102 53 | assert logger_mock.logs[0] == "Started work. Items to process: 1234." 54 | assert logger_mock.logs[1] == "99 items processed. Progress is 8%." 55 | assert logger_mock.logs[100] == "9900 items processed. Progress is 802%!!!" 56 | assert logger_mock.logs[101].startswith( 57 | "Finished work. Total items processed: 9900. Took " 58 | ) 59 | 60 | 61 | class LoggerMock: 62 | def __init__(self): 63 | self.logs = [] 64 | 65 | def info(self, message): 66 | self.logs.append(message) 67 | -------------------------------------------------------------------------------- /blockchainetl/jobs/exporters/postgres_item_exporter.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2020 Evgeny Medvedev, evge.medvedev@gmail.com 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | import collections 24 | 25 | from sqlalchemy import create_engine 26 | 27 | from blockchainetl.jobs.exporters.converters.composite_item_converter import CompositeItemConverter 28 | 29 | 30 | class PostgresItemExporter: 31 | 32 | def __init__(self, connection_url, item_type_to_insert_stmt_mapping, converters=(), print_sql=True): 33 | self.connection_url = connection_url 34 | self.item_type_to_insert_stmt_mapping = item_type_to_insert_stmt_mapping 35 | self.converter = CompositeItemConverter(converters) 36 | self.print_sql = print_sql 37 | 38 | self.engine = self.create_engine() 39 | 40 | def open(self): 41 | pass 42 | 43 | def export_items(self, items): 44 | items_grouped_by_type = group_by_item_type(items) 45 | 46 | for item_type, insert_stmt in self.item_type_to_insert_stmt_mapping.items(): 47 | item_group = items_grouped_by_type.get(item_type) 48 | if item_group: 49 | connection = self.engine.connect() 50 | converted_items = list(self.convert_items(item_group)) 51 | connection.execute(insert_stmt, converted_items) 52 | 53 | def convert_items(self, items): 54 | for item in items: 55 | yield self.converter.convert_item(item) 56 | 57 | def create_engine(self): 58 | engine = create_engine(self.connection_url, echo=self.print_sql, pool_recycle=3600) 59 | return engine 60 | 61 | def close(self): 62 | pass 63 | 64 | 65 | def group_by_item_type(items): 66 | result = collections.defaultdict(list) 67 | for item in items: 68 | result[item.get('type')].append(item) 69 | 70 | return result 71 | -------------------------------------------------------------------------------- /klaytnetl/cli/extract_token_transfers.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | import click 26 | import csv 27 | import json 28 | 29 | from blockchainetl.file_utils import smart_open 30 | from klaytnetl.jobs.exporters.token_transfers_item_exporter import ( 31 | token_transfers_item_exporter, 32 | ) 33 | from klaytnetl.jobs.extract_token_transfers_job import ExtractTokenTransfersJob 34 | from blockchainetl.logging_utils import logging_basic_config 35 | 36 | logging_basic_config() 37 | 38 | 39 | @click.command(context_settings=dict(help_option_names=["-h", "--help"])) 40 | @click.option( 41 | "-l", "--logs", type=str, required=True, help="The file containing receipt logs." 42 | ) 43 | @click.option( 44 | "-b", 45 | "--batch-size", 46 | default=100, 47 | type=int, 48 | help="The number of blocks to filter at a time.", 49 | ) 50 | @click.option( 51 | "-o", 52 | "--output", 53 | default="-", 54 | type=str, 55 | help="The output file. If not specified stdout is used.", 56 | ) 57 | @click.option( 58 | "-w", "--max-workers", default=5, type=int, help="The maximum number of workers." 59 | ) 60 | def extract_token_transfers(logs, batch_size, output, max_workers): 61 | """Extracts ERC20/ERC721/ERC1155 transfers from logs file.""" 62 | with smart_open(logs, "r") as logs_file: 63 | if logs.endswith(".json"): 64 | logs_reader = (json.loads(line) for line in logs_file) 65 | else: 66 | logs_reader = csv.DictReader(logs_file) 67 | job = ExtractTokenTransfersJob( 68 | logs_iterable=logs_reader, 69 | batch_size=batch_size, 70 | max_workers=max_workers, 71 | item_exporter=token_transfers_item_exporter(output), 72 | ) 73 | 74 | job.run() 75 | -------------------------------------------------------------------------------- /klaytnetl/jobs/exporters/blocks_and_transactions_item_exporter.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | from blockchainetl.jobs.exporters.composite_item_exporter import CompositeItemExporter 26 | 27 | BLOCK_FIELDS_TO_EXPORT = [ 28 | "number", 29 | "hash", 30 | "parent_hash", 31 | "logs_bloom", 32 | "transactions_root", 33 | "state_root", 34 | "receipts_root", 35 | "size", 36 | "extra_data", 37 | # 'gas_limit', # Does not supported by klay_getBlockWithConsensusInfoByNumber 38 | "gas_used", 39 | "block_timestamp", 40 | "block_unix_timestamp", 41 | "transaction_count", 42 | "block_score", 43 | "total_block_score", 44 | "governance_data", 45 | "vote_data", 46 | "committee", 47 | "proposer", 48 | "reward_address", 49 | "base_fee_per_gas", 50 | ] 51 | 52 | TRANSACTION_FIELDS_TO_EXPORT = [ 53 | "hash", 54 | "nonce", 55 | "block_hash", 56 | "block_number", 57 | "transaction_index", 58 | "from_address", 59 | "to_address", 60 | "value", 61 | "gas", 62 | "gas_price", 63 | "input", 64 | # Klaytn additional properties 65 | "fee_payer", 66 | "fee_payer_signatures", 67 | "fee_ratio", 68 | "sender_tx_hash", 69 | "signatures", 70 | "tx_type", 71 | "tx_type_int", 72 | "max_priority_fee_per_gas", 73 | "max_fee_per_gas", 74 | "access_list", 75 | ] 76 | 77 | 78 | def blocks_and_transactions_item_exporter(blocks_output=None, transactions_output=None): 79 | return CompositeItemExporter( 80 | filename_mapping={"block": blocks_output, "transaction": transactions_output}, 81 | field_mapping={ 82 | "block": BLOCK_FIELDS_TO_EXPORT, 83 | "transaction": TRANSACTION_FIELDS_TO_EXPORT, 84 | }, 85 | ) 86 | -------------------------------------------------------------------------------- /tests/klaytnetl/service/test_klaytn_token_transfer_extractor.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | from klaytnetl.domain.receipt_log import KlaytnReceiptLog 26 | from klaytnetl.service.token_transfer_extractor import KlaytnTokenTransferExtractor 27 | 28 | token_transfer_extractor = KlaytnTokenTransferExtractor() 29 | 30 | 31 | def test_extract_transfer_from_receipt_log(): 32 | log = KlaytnReceiptLog() 33 | log.address = "0xcee8faf64bb97a73bb51e115aa89c17ffa8dd167" 34 | log.block_number = 81165353 35 | log.data = "0x000000000000000000000000000000000000000000000000000000000501cdf5" 36 | log.log_index = 70 37 | log.topics = [ 38 | "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", 39 | "0x0000000000000000000000002bdf4c055102371aadb9b6bbe883b0b0a3a78ce0", 40 | "0x0000000000000000000000002abe3e13f3e82beb9708705164e4cc726d9802c3", 41 | ] 42 | log.transaction_hash = ( 43 | "0xf83fbed71a38ee3ce24d88ef3a60495cb88e3622ee2770a3dd74622d2ef473c6" 44 | ) 45 | log.transaction_index = 67 46 | log.block_hash = ( 47 | "0xfcb46ee2e0656c5a6da13fdd05a306f5d0cd583a2516cba95a1b492e4086c068" 48 | ) 49 | 50 | token_transfer = token_transfer_extractor.extract_transfer_from_log(log) 51 | 52 | assert token_transfer.token_address == "0xcee8faf64bb97a73bb51e115aa89c17ffa8dd167" 53 | assert token_transfer.from_address == "0x2bdf4c055102371aadb9b6bbe883b0b0a3a78ce0" 54 | assert token_transfer.to_address == "0x2abe3e13f3e82beb9708705164e4cc726d9802c3" 55 | assert token_transfer.value == 84004341 56 | assert ( 57 | token_transfer.transaction_hash 58 | == "0xf83fbed71a38ee3ce24d88ef3a60495cb88e3622ee2770a3dd74622d2ef473c6" 59 | ) 60 | assert token_transfer.block_number == 81165353 61 | -------------------------------------------------------------------------------- /tests/klaytnetl/test_utils.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | import pytest 26 | 27 | from klaytnetl.utils import int_to_decimal, float_to_datetime, validate_address 28 | from datetime import datetime, timezone 29 | from decimal import Decimal 30 | 31 | 32 | @pytest.mark.parametrize( 33 | "test_input,expected", 34 | [ 35 | (3, Decimal(3)), 36 | (Decimal(3), Decimal(3)), 37 | (None, 0), 38 | ], 39 | ) 40 | def test_int_to_decimal(test_input, expected): 41 | assert int_to_decimal(test_input) == expected 42 | 43 | 44 | @pytest.mark.parametrize( 45 | "test_input,expected", 46 | [ 47 | ( 48 | 1574770789.019, 49 | datetime(2019, 11, 26, 12, 19, 49, 19000, tzinfo=timezone.utc), 50 | ), 51 | (1574770789, datetime(2019, 11, 26, 12, 19, 49, tzinfo=timezone.utc)), 52 | ( 53 | datetime(2019, 11, 26, 12, 19, 49, 19000, tzinfo=timezone.utc), 54 | datetime(2019, 11, 26, 12, 19, 49, 19000, tzinfo=timezone.utc), 55 | ), 56 | ], 57 | ) 58 | def test_float_to_datetime(test_input, expected): 59 | assert float_to_datetime(test_input) == expected 60 | 61 | 62 | @pytest.mark.parametrize( 63 | "test_input,expected,digits", 64 | [ 65 | ( 66 | "0xc032c34cb9fe064fe435199e1078dd8756a166b5", 67 | "0xc032c34cb9fe064fe435199e1078dd8756a166b5", 68 | 42, 69 | ), 70 | ( 71 | "0x8955fe422a68babf0a83941ae18e97720ad4c2960c15e12745924af56042434c", 72 | "0x8955fe422a68babf0a83941ae18e97720ad4c2960c15e12745924af56042434c", 73 | 66, 74 | ), 75 | ], 76 | ) 77 | def test_validate_address(test_input, expected, digits): 78 | assert validate_address(test_input, digits) == expected 79 | -------------------------------------------------------------------------------- /blockchainetl/file_utils.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | 24 | import contextlib 25 | import os 26 | import pathlib 27 | import sys 28 | import gzip 29 | 30 | 31 | # https://stackoverflow.com/questions/17602878/how-to-handle-both-with-open-and-sys-stdout-nicely 32 | @contextlib.contextmanager 33 | def smart_open(filename=None, mode='w', binary=False, create_parent_dirs=True): 34 | fh = get_file_handle(filename, mode, binary, create_parent_dirs) 35 | 36 | try: 37 | yield fh 38 | finally: 39 | fh.close() 40 | 41 | 42 | def get_file_handle(filename, mode='w', binary=False, create_parent_dirs=True, compress=False): 43 | if create_parent_dirs and filename is not None: 44 | dirname = os.path.dirname(filename) 45 | pathlib.Path(dirname).mkdir(parents=True, exist_ok=True) 46 | full_mode = mode + ('b' if binary else '') 47 | is_file = filename and filename != '-' 48 | if is_file: 49 | if compress: 50 | fh = gzip.open(filename, full_mode) 51 | else: 52 | fh = open(filename, full_mode) 53 | elif filename == '-': 54 | fd = sys.stdout.fileno() if mode == 'w' else sys.stdin.fileno() 55 | fh = os.fdopen(fd, full_mode) 56 | else: 57 | fh = NoopFile() 58 | return fh 59 | 60 | 61 | def close_silently(file_handle): 62 | if file_handle is None: 63 | pass 64 | try: 65 | file_handle.close() 66 | except OSError: 67 | pass 68 | 69 | 70 | class NoopFile: 71 | def __enter__(self): 72 | pass 73 | 74 | def __exit__(self): 75 | pass 76 | 77 | def readable(self): 78 | pass 79 | 80 | def writable(self): 81 | pass 82 | 83 | def seekable(self): 84 | pass 85 | 86 | def close(self): 87 | pass 88 | 89 | def write(self, bytes): 90 | pass 91 | -------------------------------------------------------------------------------- /klaytnetl/cli/extract_contracts.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | import csv 26 | import json 27 | 28 | import click 29 | from blockchainetl.csv_utils import set_max_field_size_limit 30 | from blockchainetl.file_utils import smart_open 31 | from klaytnetl.jobs.exporters.contracts_item_exporter import contracts_item_exporter 32 | from klaytnetl.jobs.extract_contracts_job import ExtractContractsJob 33 | from blockchainetl.logging_utils import logging_basic_config 34 | 35 | logging_basic_config() 36 | 37 | 38 | @click.command(context_settings=dict(help_option_names=["-h", "--help"])) 39 | @click.option( 40 | "-t", "--traces", type=str, required=True, help="The CSV file containing traces." 41 | ) 42 | @click.option( 43 | "-b", 44 | "--batch-size", 45 | default=100, 46 | type=int, 47 | help="The number of blocks to filter at a time.", 48 | ) 49 | @click.option( 50 | "-o", 51 | "--output", 52 | default="-", 53 | type=str, 54 | help="The output file. If not specified stdout is used.", 55 | ) 56 | @click.option( 57 | "-w", "--max-workers", default=5, type=int, help="The maximum number of workers." 58 | ) 59 | def extract_contracts(traces, batch_size, output, max_workers): 60 | """Extracts contracts from traces file.""" 61 | 62 | set_max_field_size_limit() 63 | 64 | with smart_open(traces, "r") as traces_file: 65 | if traces.endswith(".json"): 66 | traces_iterable = (json.loads(line) for line in traces_file) 67 | else: 68 | traces_iterable = csv.DictReader(traces_file) 69 | job = ExtractContractsJob( 70 | traces_iterable=traces_iterable, 71 | batch_size=batch_size, 72 | max_workers=max_workers, 73 | item_exporter=contracts_item_exporter(output), 74 | ) 75 | 76 | job.run() 77 | -------------------------------------------------------------------------------- /klaytnetl/jobs/extract_token_transfers_job.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | from klaytnetl.executors.batch_work_executor import BatchWorkExecutor 26 | from blockchainetl.jobs.base_job import BaseJob 27 | from klaytnetl.mappers.token_transfer_mapper import KlaytnTokenTransferMapper 28 | from klaytnetl.mappers.receipt_log_mapper import KlaytnReceiptLogMapper 29 | from klaytnetl.service.token_transfer_extractor import KlaytnTokenTransferExtractor 30 | 31 | 32 | class ExtractTokenTransfersJob(BaseJob): 33 | def __init__(self, logs_iterable, batch_size, max_workers, item_exporter): 34 | self.logs_iterable = logs_iterable 35 | 36 | self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) 37 | self.item_exporter = item_exporter 38 | 39 | self.receipt_log_mapper = KlaytnReceiptLogMapper() 40 | self.token_transfer_mapper = KlaytnTokenTransferMapper() 41 | self.token_transfer_extractor = KlaytnTokenTransferExtractor() 42 | 43 | def _start(self): 44 | self.item_exporter.open() 45 | 46 | def _export(self): 47 | self.batch_work_executor.execute(self.logs_iterable, self._extract_transfers) 48 | 49 | def _extract_transfers(self, log_dicts): 50 | for log_dict in log_dicts: 51 | self._extract_transfer(log_dict) 52 | 53 | def _extract_transfer(self, log_dict): 54 | log = self.receipt_log_mapper.dict_to_receipt_log(log_dict) 55 | token_transfer = self.token_transfer_extractor.extract_transfer_from_log(log) 56 | if token_transfer is not None: 57 | self.item_exporter.export_item( 58 | self.token_transfer_mapper.token_transfer_to_dict(token_transfer) 59 | ) 60 | 61 | def _end(self): 62 | self.batch_work_executor.shutdown() 63 | self.item_exporter.close() 64 | -------------------------------------------------------------------------------- /klaytnetl/trace_progress_logger.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Modifications Copyright (c) klaytn authors 4 | # Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | from klaytnetl.progress_logger import ProgressLogger 26 | 27 | 28 | class TraceProgressLogger(ProgressLogger): 29 | def __init__(self, trace_count=0, name="work", logger=None, log_percentage_step=10, log_item_step=5000): 30 | super().__init__(name, logger, log_percentage_step, log_item_step) 31 | self.trace_count = trace_count 32 | 33 | def __getattr__(self, attr): 34 | return getattr(self.obj, attr) 35 | 36 | def track(self, item_count=1, trace_count=0): 37 | processed_items = self.counter.increment(item_count) 38 | processed_items_before = processed_items - item_count 39 | 40 | track_message = None 41 | 42 | if trace_count: 43 | self.trace_count += trace_count 44 | 45 | current_trace_count = self.trace_count 46 | if self.total_items is None: 47 | if int(processed_items_before / self.log_items_step) != int( 48 | processed_items / self.log_items_step 49 | ): 50 | track_message = "{} items processed.".format(processed_items) 51 | else: 52 | percentage = processed_items * 100 / self.total_items 53 | percentage_before = processed_items_before * 100 / self.total_items 54 | if int(percentage_before / self.log_percentage_step) != int( 55 | percentage / self.log_percentage_step 56 | ): 57 | track_message = "{} items processed. Block Progress is {}%, Trace Count is {}".format( 58 | processed_items, int(percentage), current_trace_count 59 | ) + ("!!!" if int(percentage) > 100 else ".") 60 | self.trace_count = 0 61 | 62 | if track_message is not None: 63 | self.logger.info(track_message) 64 | -------------------------------------------------------------------------------- /schemas/contracts.py: -------------------------------------------------------------------------------- 1 | CONTRACTS_SCHEMA = [ 2 | { 3 | "name": "address", 4 | "type": "STRING", 5 | "mode": "NULLABLE", 6 | "description": "Address of the contract", 7 | }, 8 | { 9 | "name": "bytecode", 10 | "type": "STRING", 11 | "mode": "NULLABLE", 12 | "description": "Bytecode of the contract", 13 | }, 14 | { 15 | "name": "function_sighashes", 16 | "type": "STRING", 17 | "mode": "REPEATED", 18 | "description": "4-byte function signature hashes", 19 | }, 20 | { 21 | "name": "is_erc20", 22 | "type": "BOOLEAN", 23 | "mode": "NULLABLE", 24 | "description": "Whether this contract is an ERC20 contract", 25 | }, 26 | { 27 | "name": "is_erc721", 28 | "type": "BOOLEAN", 29 | "mode": "NULLABLE", 30 | "description": "Whether this contract is an ERC721 contract", 31 | }, 32 | { 33 | "name": "is_erc1155", 34 | "type": "BOOLEAN", 35 | "mode": "NULLABLE", 36 | "description": "Whether this contract is an ERC1155 contract", 37 | }, 38 | { 39 | "name": "block_number", 40 | "type": "INT64", 41 | "mode": "NULLABLE", 42 | "description": "Block number corresponding", 43 | }, 44 | { 45 | "name": "block_hash", 46 | "type": "STRING", 47 | "mode": "NULLABLE", 48 | "description": "Hash of the block", 49 | }, 50 | { 51 | "name": "block_timestamp", 52 | "type": "TIMESTAMP", 53 | "mode": "NULLABLE", 54 | "description": "The UTC timestamp for when the block was collated", 55 | }, 56 | { 57 | "name": "block_unix_timestamp", 58 | "type": "FLOAT64", 59 | "mode": "NULLABLE", 60 | "description": "The unix timestamp for when the block was collated", 61 | }, 62 | { 63 | "name": "transaction_hash", 64 | "type": "STRING", 65 | "mode": "NULLABLE", 66 | "description": "Hash of the transactions", 67 | }, 68 | { 69 | "name": "transaction_index", 70 | "type": "INT64", 71 | "mode": "NULLABLE", 72 | "description": "Integer of the transactions index position in the block", 73 | }, 74 | { 75 | "name": "transaction_receipt_status", 76 | "type": "INT64", 77 | "mode": "NULLABLE", 78 | "description": "Either 1 (success) or 0 (failure) (post Byzantium)", 79 | }, 80 | { 81 | "name": "trace_index", 82 | "type": "INT64", 83 | "mode": "NULLABLE", 84 | "description": "Index of the trace", 85 | }, 86 | { 87 | "name": "trace_status", 88 | "type": "INT64", 89 | "mode": "NULLABLE", 90 | "description": "Either 1 (success) or 0 (failure, due to any operation that can cause the call itself or any top-level call to revert)", 91 | }, 92 | { 93 | "name": "creator_address", 94 | "type": "STRING", 95 | "mode": "NULLABLE", 96 | "description": "Token creator address", 97 | }, 98 | ] 99 | -------------------------------------------------------------------------------- /tests/resources/test_export_receipts_job/receipts/web3_response.klay_getTransactionReceipt_0x5778fac21d744201bf0cce8c014207938f55eb9213989086f3f2228acbac4b54.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":{"blockHash":"0x27b0905c69110e2f9a88c6fbbe8f947f62d2db388fff600c0940ffb685f55fc3","blockNumber":"0x5d96c13","contractAddress":null,"effectiveGasPrice":"0x3a35294400","feePayer":"0x10f0eaa52a474f0605afbb8c17bb9189e6ed006c","feePayerSignatures":[{"V":"0x4055","R":"0x41f41baac7d391b3d5bc2a171e2fc3645f0ddabc6aca3b17cdba73802bdd2100","S":"0x4aeb88f87bbd8f87b2f1b2b9a00bd9ac043c1f8145867c429fe21c45700a2c22"}],"from":"0xe0c9c6fb8e8c039b118b32e74d680f84d8c885e3","gas":"0x13a74","gasPrice":"0x3a35294400","gasUsed":"0x13a74","input":"0xf901ec8180b901e77b22626c6f636b436f756e74223a3330302c22626c6f636b48617368223a22307830346631336334326135663765323239333035633039323139373831643233653564393365356136303634616566653263386639353637323831336162383566222c22626c6f636b4e756d626572223a323737343130302c226964223a2232373734313030222c22706172656e7448617368223a22307861333536396462396637613436316638306135633334663632653136393234303337393663343266313231323634313032616236626362313361363365323338222c227265636569707473526f6f74223a22307831633131626336356135353864306236626136646239343836656666313764326436623832653865353733393233376166323734613063353738313539633734222c227374617465526f6f74223a22307863633661643131313166393036663632366161363238613663646138623439396662346339313664366638666236386133383164366231343830326237343836222c227472616e73616374696f6e73526f6f74223a22307837326138646239383431346636363062643961356638386339373937643966316434313666303666616438636462303861376162323235633133666566626330222c227478436f756e74223a31383339327d","inputJSON":{"blockCount":300,"blockHash":"0x04f13c42a5f7e229305c09219781d23e5d93e5a6064aefe2c8f95672813ab85f","blockNumber":2774100,"id":"2774100","parentHash":"0xa3569db9f7a461f80a5c34f62e1692403796c42f121264102ab6bcb13a63e238","receiptsRoot":"0x1c11bc65a558d0b6ba6db9486eff17d2d6b82e8e5739237af274a0c578159c74","stateRoot":"0xcc6ad1111f906f626aa628a6cda8b499fb4c916d6f8fb68a381d6b14802b7486","transactionsRoot":"0x72a8db98414f660bd9a5f88c9797d9f1d416f06fad8cdb08a7ab225c13fefbc0","txCount":18392},"logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","nonce":"0x241d","senderTxHash":"0x1140b9d1ae8c2cf8039cf9af4bb6b53f6ebb5dadf9b7e3e838c2747da4084f9a","signatures":[{"V":"0x4055","R":"0x8770b72109ba430a3d9adf768cd643527f60243c7850399d85a60d4996afcd06","S":"0x294ee9fa9e51070ebb7019c46b1c598cf23701fe91f92af577473ba0acef8457"}],"status":"0x1","transactionHash":"0x5778fac21d744201bf0cce8c014207938f55eb9213989086f3f2228acbac4b54","transactionIndex":"0x1","type":"TxTypeFeeDelegatedChainDataAnchoring","typeInt":73}} -------------------------------------------------------------------------------- /tests/resources/test_export_receipts_job/receipts/web3_response.klay_getTransactionReceipt_0x5eb066bd544519dd77657983d2daf67183bd9ad2c7c581978746d9a27f6a79f3.json: -------------------------------------------------------------------------------- 1 | {"jsonrpc":"2.0","id":1,"result":{"blockHash":"0x9510a4d60ff038a3c875a0bdc45e20e09f8967989bb0cc69be3065dad3e1622d","blockNumber":"0x5d97314","contractAddress":null,"effectiveGasPrice":"0x3a35294400","feePayer":"0x67abbf32db2bdc982b6b8782d8b4dea36c6115e5","feePayerSignatures":[{"V":"0x4055","R":"0xcd10909b5d50593d2ee7e06795487d55cb4285769114d60016e39fec39c23684","S":"0x2e0cbd6ab39b3c6387040465bc5b127569b370e9c20c65aada7a6b7a56fb970a"}],"from":"0xf366f718add52e0710e8e3c67365a6d4f51d43ec","gas":"0x419ce0","gasPrice":"0x3a35294400","gasUsed":"0x1607d","input":"0xeb7955490000000000000000000000006c8db4fef09fe9ab0238f13419e503c78e421492000000000000000000000000000000000000000000000007ccdff8a35af50000000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000000000000000000150b2e7b5e747aa1b3a93e17a6a24c1e800ddae1e6a20000000000000000000000","logs":[{"address":"0xdcd62c57182e780e23d2313c4782709da85b9d6c","topics":["0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef","0x000000000000000000000000f366f718add52e0710e8e3c67365a6d4f51d43ec","0x0000000000000000000000006c8db4fef09fe9ab0238f13419e503c78e421492"],"data":"0x000000000000000000000000000000000000000000000007ccdff8a35af50000","blockNumber":"0x5d97314","transactionHash":"0x5eb066bd544519dd77657983d2daf67183bd9ad2c7c581978746d9a27f6a79f3","transactionIndex":"0x0","blockHash":"0x9510a4d60ff038a3c875a0bdc45e20e09f8967989bb0cc69be3065dad3e1622d","logIndex":"0x0","removed":false},{"address":"0x6c8db4fef09fe9ab0238f13419e503c78e421492","topics":["0x6eb1790ea00e2fbc3f2b2da832350b4306d65c4114874a9f0512527d33da62e4"],"data":"0x0000000000000000000000002e7b5e747aa1b3a93e17a6a24c1e800ddae1e6a2000000000000000000000000000000000000000000000007ccdff8a35af50000","blockNumber":"0x5d97314","transactionHash":"0x5eb066bd544519dd77657983d2daf67183bd9ad2c7c581978746d9a27f6a79f3","transactionIndex":"0x0","blockHash":"0x9510a4d60ff038a3c875a0bdc45e20e09f8967989bb0cc69be3065dad3e1622d","logIndex":"0x1","removed":false}],"logsBloom":"0x00000000000000000000000000000000000000020000000000000800001000000000000000000000000000000000000000000000000000000000000000000000000200000000000000000008000000000000000000000000000000000000000000000000000000000000000000000000000000000000000840000010000000000000000000000000000000000000000000000000000000000000800080000000000000000000000000002000000000000000000000000000000000000000000200000002000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000001000000010004040000000000000","nonce":"0x2388a9","senderTxHash":"0xef34cfaff76e3b32b70357a0434132d54e3b6ae88eccec8f30356b9a10787784","signatures":[{"V":"0x4055","R":"0x27ec8cccc9a8dbaa8b87d263787dda7b7663d3702c519e1888a50ea4cd1a790","S":"0x790df191354a29cd2f89b85caf33a369517d69ae9df04ff2cbca6062da5260aa"}],"status":"0x1","to":"0xdcd62c57182e780e23d2313c4782709da85b9d6c","transactionHash":"0x5eb066bd544519dd77657983d2daf67183bd9ad2c7c581978746d9a27f6a79f3","transactionIndex":"0x0","type":"TxTypeFeeDelegatedSmartContractExecution","typeInt":49,"value":"0x0"}} --------------------------------------------------------------------------------