├── nebulachia ├── __init__.py └── convertor.py ├── requirements.txt ├── .gitignore ├── setup.py ├── README.md └── nebula-importer └── nebula-chia.yaml /nebulachia/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | chia-blockchain -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | .DS_Store 3 | 4 | build/ 5 | dist/ 6 | *.egg-info/ 7 | 8 | __pycache__ -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r", encoding="utf-8") as fh: 4 | long_description = fh.read() 5 | 6 | setuptools.setup( 7 | name="nebula-chia", 8 | version="0.2", 9 | author="Wey Gu", 10 | author_email="weyl.gu@gmail.com", 11 | description="Chia Network data ETL for Nebula Graph", 12 | long_description=long_description, 13 | long_description_content_type="text/markdown", 14 | url="https://github.com/wey-gu/nebula-chia", 15 | project_urls={ 16 | "Bug Tracker": "https://github.com/wey-gu/nebula-chia/issues", 17 | }, 18 | classifiers=[ 19 | "Programming Language :: Python :: 3", 20 | "License :: OSI Approved :: Apache Software License", 21 | "Operating System :: OS Independent", 22 | ], 23 | packages=setuptools.find_packages(), 24 | python_requires=">=3.6", 25 | install_requires=[ 26 | 'chia-blockchain', 27 | ], 28 | ) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # nebula-chia 2 | 3 | ## How To Use 4 | 5 | ### ChaiBatchConvertor 6 | 7 | #### Step 0, Installation 8 | 9 | `nebula-chia` could be installed either via pip or from this git repo itself. 10 | 11 | > Install via pip 12 | 13 | ```bash 14 | python3 -m pip install nebula-chia 15 | ``` 16 | > Install from the github repo 17 | 18 | ```bash 19 | git clone git@github.com:wey-gu/nebula-chia.git 20 | cd nebula-chia 21 | python3 setup.py install 22 | ``` 23 | > Note: 24 | > 25 | > Nebula-chia depends on `chia-blockchain`, the easiest way is to call it from the venv inside the Chia Network repo. 26 | > 27 | > - Install Chia Netowrk refers to https://github.com/Chia-Network/chia-blockchain/wiki/INSTALL 28 | > 29 | > - Activate venv like: 30 | > 31 | > - ```bash 32 | > cd chia-blockchain 33 | > . ./activate 34 | > ``` 35 | 36 | #### Step 1, Convert Chia as CSV files 37 | 38 | `ChiaBatchConvertor` is used to convert Chia Block Chain data into CSV files, which could then be used for nebula-importer 39 | 40 | ```bash 41 | $ python3 -m pip install nebula-chia 42 | $ python 43 | 44 | # block_record_limit = 0 means unlimited 45 | # coin_record_limit = 0 means unlimited 46 | from nebulachia.convertor import ChiaBatchConvertor 47 | c = ChaiBatchConvertor(block_record_limit=0, coin_record_limit=0, write_batch_size=10000) 48 | c.convert_block_record() 49 | c.convert_coin_record() 50 | exit() 51 | 52 | $ ls -lth 53 | 54 | -rw-r--r-- 1 weyl staff 173M May 19 13:01 coin_record.csv 55 | -rw-r--r-- 1 weyl staff 77M May 19 12:59 block_record.csv 56 | ... 57 | 58 | ``` 59 | 60 | #### Step2, Import CSV files to Nebula Graph 61 | 62 | In above steps, we already have `coin_record.csv` and `block_record.csv` generated, now we could import the data into a nebula graph cluster with the help of [Nebula-Importer](https://github.com/vesoft-inc/nebula-importer/). 63 | 64 | The `nebula-chia.yaml` in this repo is the config file for nebula-importer. 65 | 66 | ```bash 67 | ❯ tree nebula-importer 68 | nebula-importer 69 | └── nebula-chia.yaml 70 | ``` 71 | 72 | Below is an example of running importer, which assumed both our CSV files and the `nebula-chia.yaml` placed in path `/home/nebula/chia/`. 73 | 74 | ```bash 75 | docker run --rm -ti \ 76 | --network=nebula-docker-compose_nebula-net \ 77 | -v /home/nebula/chia:/root \ 78 | vesoft/nebula-importer:v2 \ 79 | --config /root/nebula-chia.yaml 80 | ``` 81 | 82 | -------------------------------------------------------------------------------- /nebulachia/convertor.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sqlite3 3 | 4 | from chia.consensus.block_record import BlockRecord 5 | from chia.util.ints import uint32, uint64 6 | 7 | 8 | DEFAULT_DB_PATH = "~/.chia/mainnet/db/blockchain_v1_mainnet.sqlite" 9 | DEFAULT_BLOCK_RECORD_LIMIT = 10000 10 | DEFAULT_COIN_RECORD_LIMIT = 10000 11 | 12 | BLOCKRECORD_FILENAME = "block_record.csv" 13 | COINRECORD_FILENAME = "coin_record.csv" 14 | DEFAULT_OUTPUT_PATH = os.getcwd() 15 | CSV_WRITE_BATCH = 10000 16 | 17 | 18 | class ChiaBatchConvertor: 19 | def __init__(self, db_path=None, output_path=None, 20 | block_record_limit=None, coin_record_limit=None, 21 | write_batch_size=None): 22 | """ 23 | Initilization for ChiaBatchConvertor. 24 | """ 25 | if db_path is None: 26 | db_path = os.path.expanduser(DEFAULT_DB_PATH) 27 | self.db_path = db_path 28 | self.db_con = sqlite3.connect(self.db_path) 29 | 30 | if output_path is None: 31 | output_path = DEFAULT_OUTPUT_PATH 32 | self.output_path = output_path 33 | 34 | if block_record_limit is None: 35 | block_record_limit = DEFAULT_BLOCK_RECORD_LIMIT 36 | self.block_limit = block_record_limit 37 | 38 | if coin_record_limit is None: 39 | coin_record_limit = DEFAULT_COIN_RECORD_LIMIT 40 | self.coin_limit = coin_record_limit 41 | 42 | if write_batch_size is None: 43 | write_batch_size = CSV_WRITE_BATCH 44 | self.write_batch_size = write_batch_size 45 | 46 | def convert_block_record(self): 47 | """ 48 | Parse block record and convert the record into CSV lines. 49 | """ 50 | file_path = f"{ self.output_path }/{ BLOCKRECORD_FILENAME }" 51 | block_record_query = "SELECT * FROM block_records" 52 | 53 | if self.block_limit > 0: 54 | block_record_query = block_record_query + f" LIMIT { self.block_limit }" 55 | cur = self.db_con.cursor() 56 | rows_iterator = cur.execute(block_record_query) 57 | 58 | self._csv_writer( 59 | file_path, rows_iterator, 60 | self.block_record_row, self.write_batch_size) 61 | 62 | 63 | def convert_coin_record(self): 64 | """ 65 | Parse coin record and convert the record into CSV lines. 66 | """ 67 | file_path = f"{ self.output_path }/{ COINRECORD_FILENAME }" 68 | coin_record_query = "SELECT * FROM coin_record" 69 | 70 | if self.coin_limit > 0: 71 | coin_record_query = coin_record_query + f" LIMIT { self.coin_limit }" 72 | cur = self.db_con.cursor() 73 | rows_iterator = cur.execute(coin_record_query) 74 | 75 | self._csv_writer( 76 | file_path, rows_iterator, 77 | self.coin_record_row, self.write_batch_size) 78 | 79 | @staticmethod 80 | def _csv_writer(file_path, iterator, row_formator, buffer_size): 81 | with open(file_path, mode='w') as file: 82 | writer = csv.writer( 83 | file, delimiter=',',quotechar='|', quoting=csv.QUOTE_MINIMAL) 84 | csv_buffer = list() 85 | for row in iterator: 86 | csv_buffer.append(row_formator(row)) 87 | if len(csv_buffer) > buffer_size: 88 | writer.writerows(csv_buffer) 89 | del csv_buffer[:] 90 | if csv_buffer: 91 | writer.writerows(csv_buffer) 92 | del csv_buffer[:] 93 | 94 | @staticmethod 95 | def _to_bool(value): 96 | """ 97 | tinyint to string bool in lower case. 98 | """ 99 | return "false" if value in [0, "0"] else "true" 100 | 101 | def block_record_row(self, row): 102 | """ 103 | Parse row and return a CSV block record row list. 104 | 105 | CREATE TABLE block_records( 106 | header_hash text PRIMARY KEY, 107 | prev_hash text, 108 | height bigint, 109 | block blob, 110 | sub_epoch_summary blob, 111 | is_peak tinyint, 112 | is_block tinyint) 113 | 114 | Block Record CSV Head: 115 | 0 1 2(int) 3(bool) 4(bool) 116 | header_hash|prev_hash|height|is_peak|is_block| 117 | 118 | 5(int) 119 | deficit| 120 | 121 | 6 122 | challenge_block_info_hash| 123 | 124 | 7 125 | farmer_puzzle_hash| 126 | 127 | 8(int) 128 | fees| 129 | 130 | 9 131 | prev_transaction_block_hash| 132 | 133 | 10 134 | prev_transaction_block_height| 135 | 136 | 11 12(int) 137 | required_iters|signage_point_index| 138 | 139 | 13(timestamp) 140 | timestamp 141 | """ 142 | rec = BlockRecord.from_bytes(row[3]) 143 | row_list = ( 144 | row[0], row[1], str(row[2]), self._to_bool(row[5]), self._to_bool(row[6]), 145 | str(rec.deficit.real), 146 | str(rec.challenge_block_info_hash) if rec.challenge_block_info_hash else str(), 147 | str(rec.farmer_puzzle_hash) if rec.farmer_puzzle_hash else str(), 148 | str(0 if rec.fees is None else rec.fees), 149 | str(rec.prev_transaction_block_hash) if rec.prev_transaction_block_hash else str(), 150 | str(rec.prev_transaction_block_height), 151 | str(rec.required_iters), str(rec.signage_point_index), 152 | str(0 if rec.timestamp is None else rec.timestamp) 153 | ) 154 | return row_list 155 | 156 | def _height_to_hash(self, height): 157 | cur = self.db_con.cursor() 158 | if height == 0: 159 | return "0" 160 | query_string = f"SELECT * FROM block_records WHERE height = { height }" 161 | query_result = list(cur.execute(query_string)) 162 | if query_result: 163 | return query_result[0][1] 164 | else: 165 | print(f"[ERROR] failed during { query_string }") 166 | raise 167 | 168 | def coin_record_row(self, row): 169 | """ 170 | Parse row and return a CSV block coin row list. 171 | 172 | CREATE TABLE coin_record( 173 | coin_name text PRIMARY KEY, 174 | confirmed_index bigint, 175 | spent_index bigint, 176 | spent int, 177 | coinbase int, 178 | puzzle_hash text, 179 | coin_parent text, 180 | amount blob, 181 | timestamp bigint) 182 | 183 | Coin Record CSV Head: 184 | 0 1(int) 2(int) 3(bool) 185 | coin_name|confirmed_index|spent_index|spent| 186 | 187 | 4(bool) 5 6 7(int) 188 | coinbase|puzzle_hash|coin_parent|amount| 189 | 190 | 8(timestamp) 191 | timestamp| 192 | 193 | 9 10 194 | confirmed_hash|spent_hash 195 | """ 196 | row_list = ( 197 | row[0], str(row[1]), str(row[2]), self._to_bool(row[3]), 198 | self._to_bool(row[4]), row[5], row[6], str(uint64.from_bytes(row[7])), 199 | str(row[8]), 200 | self._height_to_hash(row[1]), self._height_to_hash(row[2]) 201 | ) 202 | return row_list 203 | -------------------------------------------------------------------------------- /nebula-importer/nebula-chia.yaml: -------------------------------------------------------------------------------- 1 | version: v2 2 | description: nebula-chia importer config file 3 | removeTempFiles: false 4 | clientSettings: 5 | retry: 3 6 | concurrency: 3 # number of graph clients 7 | channelBufferSize: 1 8 | space: chia 9 | connection: 10 | user: root 11 | password: nebula 12 | address: graphd:9669 13 | postStart: 14 | commands: | 15 | DROP SPACE IF EXISTS chia; 16 | CREATE SPACE IF NOT EXISTS chia(partition_num=10, replica_factor=1, vid_type=FIXED_STRING(64)); 17 | USE chia; 18 | CREATE TAG block(height int, is_peak bool, is_block bool, deficit int, fees int, required_iters int, signage_point_index int, block_timestamp int); 19 | CREATE TAG coin(confirmed_index int, spent_index int, is_spent bool, is_coinbase bool, amount int, coin_timestamp int); 20 | CREATE TAG puzzle(); 21 | CREATE EDGE prev_block(); 22 | CREATE EDGE prev_tran_block(); 23 | CREATE EDGE challenge_block(); 24 | CREATE EDGE farmer_puzzle(); 25 | CREATE EDGE spends(); 26 | CREATE EDGE confirms(); 27 | CREATE EDGE belongs_to(); 28 | CREATE EDGE child_of(); 29 | afterPeriod: 8s 30 | preStop: 31 | commands: | 32 | UPDATE CONFIGS storage:rocksdb_column_family_options=false; 33 | UPDATE CONFIGS storage:wal_ttl=86400; 34 | logPath: ./err/nebula-chia-importer.log 35 | files: 36 | - path: ./block_record.csv 37 | failDataPath: ./err/block_record.csv 38 | batchSize: 768 39 | inOrder: true 40 | type: csv 41 | csv: 42 | withHeader: false 43 | withLabel: false 44 | schema: 45 | type: vertex 46 | vertex: 47 | vid: 48 | index: 0 49 | tags: 50 | - name: block 51 | props: 52 | - name: height 53 | type: int 54 | index: 2 55 | - name: is_peak 56 | type: bool 57 | index: 3 58 | - name: is_block 59 | type: bool 60 | index: 4 61 | - name: deficit 62 | type: int 63 | index: 5 64 | - name: fees 65 | type: int 66 | index: 8 67 | - name: required_iters 68 | type: int 69 | index: 11 70 | - name: signage_point_index 71 | type: int 72 | index: 12 73 | - name: block_timestamp 74 | type: int 75 | index: 13 76 | 77 | - path: ./coin_record.csv 78 | failDataPath: ./err/coin_record_coin.csv 79 | batchSize: 768 80 | inOrder: true 81 | type: csv 82 | csv: 83 | withHeader: false 84 | withLabel: false 85 | schema: 86 | type: vertex 87 | vertex: 88 | vid: 89 | index: 0 90 | tags: 91 | - name: coin 92 | props: 93 | - name: confirmed_index 94 | type: int 95 | index: 1 96 | - name: spent_index 97 | type: int 98 | index: 2 99 | - name: is_spent 100 | type: int 101 | index: 3 102 | - name: is_coinbase 103 | type: bool 104 | index: 4 105 | - name: amount 106 | type: bool 107 | index: 7 108 | - name: coin_timestamp 109 | type: int 110 | index: 8 111 | 112 | - path: ./coin_record.csv 113 | failDataPath: ./err/coin_record_puzzle.csv 114 | batchSize: 768 115 | inOrder: true 116 | type: csv 117 | csv: 118 | withHeader: false 119 | withLabel: false 120 | schema: 121 | type: vertex 122 | vertex: 123 | vid: 124 | index: 0 125 | tags: 126 | - name: puzzle 127 | 128 | - path: ./block_record.csv 129 | failDataPath: ./err/block_record_prev_block.csv 130 | batchSize: 768 131 | inOrder: true 132 | type: csv 133 | csv: 134 | withHeader: false 135 | withLabel: false 136 | schema: 137 | type: edge 138 | edge: 139 | name: prev_block 140 | withRanking: false 141 | srcVID: 142 | index: 0 143 | dstVID: 144 | index: 1 145 | 146 | - path: ./block_record.csv 147 | failDataPath: ./err/block_record_prev_tran_block.csv 148 | batchSize: 768 149 | inOrder: true 150 | type: csv 151 | csv: 152 | withHeader: false 153 | withLabel: false 154 | schema: 155 | type: edge 156 | edge: 157 | name: prev_tran_block 158 | withRanking: false 159 | srcVID: 160 | index: 0 161 | dstVID: 162 | index: 9 163 | 164 | - path: ./block_record.csv 165 | failDataPath: ./err/block_record_challenge_block.csv 166 | batchSize: 768 167 | inOrder: true 168 | type: csv 169 | csv: 170 | withHeader: false 171 | withLabel: false 172 | schema: 173 | type: edge 174 | edge: 175 | name: challenge_block 176 | withRanking: false 177 | srcVID: 178 | index: 0 179 | dstVID: 180 | index: 6 181 | 182 | - path: ./block_record.csv 183 | failDataPath: ./err/block_record_farmer_puzzle.csv 184 | batchSize: 768 185 | inOrder: true 186 | type: csv 187 | csv: 188 | withHeader: false 189 | withLabel: false 190 | schema: 191 | type: edge 192 | edge: 193 | name: farmer_puzzle 194 | withRanking: false 195 | srcVID: 196 | index: 0 197 | dstVID: 198 | index: 7 199 | 200 | - path: ./coin_record.csv 201 | failDataPath: ./err/coin_record_spends.csv 202 | batchSize: 768 203 | inOrder: true 204 | type: csv 205 | csv: 206 | withHeader: false 207 | withLabel: false 208 | schema: 209 | type: edge 210 | edge: 211 | name: spends 212 | withRanking: false 213 | srcVID: 214 | index: 10 215 | dstVID: 216 | index: 0 217 | 218 | - path: ./coin_record.csv 219 | failDataPath: ./err/coin_record_confirms.csv 220 | batchSize: 768 221 | inOrder: true 222 | type: csv 223 | csv: 224 | withHeader: false 225 | withLabel: false 226 | schema: 227 | type: edge 228 | edge: 229 | name: confirms 230 | withRanking: false 231 | srcVID: 232 | index: 9 233 | dstVID: 234 | index: 0 235 | 236 | - path: ./coin_record.csv 237 | failDataPath: ./err/coin_record_child_of.csv 238 | batchSize: 768 239 | inOrder: true 240 | type: csv 241 | csv: 242 | withHeader: false 243 | withLabel: false 244 | schema: 245 | type: edge 246 | edge: 247 | name: child_of 248 | withRanking: false 249 | srcVID: 250 | index: 0 251 | dstVID: 252 | index: 6 253 | 254 | - path: ./coin_record.csv 255 | failDataPath: ./err/coin_record_belongs_to.csv 256 | batchSize: 768 257 | inOrder: true 258 | type: csv 259 | csv: 260 | withHeader: false 261 | withLabel: false 262 | schema: 263 | type: edge 264 | edge: 265 | name: belongs_to 266 | withRanking: false 267 | srcVID: 268 | index: 0 269 | dstVID: 270 | index: 5 271 | --------------------------------------------------------------------------------