├── .gitignore ├── LICENSE ├── README.md ├── TxSpectorTranslator ├── 0x37085f336b5d3e588e37674544678f8cb0fc092a6de5d83bd647e20e5232897b.txt ├── Trace2InvTranslated.txt └── translator.py ├── constraintPackage ├── README.md ├── accessControlInfer.py ├── dataFlowInfer.py ├── gasControlInfer.py ├── moneyFlowInfer.py ├── oracleControl.py ├── reentrancyInfer.py ├── specialStorage.py └── timeLockInfer.py ├── crawlPackage ├── .DS_Store ├── README.md ├── __init__.py ├── cache │ └── cacheDeployTx.pickle ├── cacheDatabase.py ├── crawl.py ├── crawlEtherscan.py ├── crawlQuicknode.py ├── crawlTrueBlocks.py └── database │ └── etherScan.db ├── fetchPackage ├── StackCarpenter.py ├── __init__.py └── fetchTrace.py ├── images ├── Comparison.png └── Overview.png ├── main.py ├── parserPackage ├── README.md ├── dataSource.py ├── decoder.py ├── functions.py ├── locator.py ├── parser.py ├── parserGlobal.py └── traceTree.py ├── settings.toml ├── staticAnalyzer ├── README.md ├── __init__.py ├── analyzer.py ├── cache │ ├── 0x39aa39c021dfbae8fac545936693ac917d5e7563_storageMapping.pickle │ ├── 0x3bc6aa2d25313ad794b2d67f83f21d341cc3f5fb_storageMapping.pickle │ ├── 0x3d9819210a31b4961b30ef54be2aed79b9c9cd3b_storageMapping.pickle │ ├── 0x75442ac771a7243433e033f3f8eab2631e22938f_storageMapping.pickle │ ├── 0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48_storageMapping.pickle │ ├── 0xa2327a938febf5fec13bacfb16ae10ecbc4cbdcf_storageMapping.pickle │ ├── 0xb650eb28d35691dd1bd481325d40e65273844f9b_storageMapping.pickle │ └── 0xd8ec56013ea119e7181d231e5048f90fbbe753c0_storageMapping.pickle ├── slitherAnalyzer.py ├── storageMappings │ └── README.md ├── temp.txt ├── temp2.txt └── vyperAnalyzer.py ├── trackerPackage ├── dataSource.py ├── memoryTracker.py ├── stackTracker.py ├── storageTracker.py └── tracker.py └── utilsPackage ├── compressor.py └── tomlHandler.py /.gitignore: -------------------------------------------------------------------------------- 1 | */__pycache__/* 2 | crawlPackage/__pycache__/* 3 | fetchPackage/__pycache__/* 4 | *.pyc 5 | crytic-export/* 6 | staticAnalyzer/slither 7 | # staticAnalyzer/cache/ 8 | 9 | 10 | # where we store the cache of etherscan/quicknode receipts 11 | crawlPackage/database/* 12 | crawlPackage/cache/ 13 | 14 | settings.toml 15 | 16 | # where we store the cache of transactions 17 | cache/ 18 | 19 | playground.ipynb 20 | *.pyc 21 | 22 | 23 | *.pickle 24 | 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Zhiyang Chen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /TxSpectorTranslator/translator.py: -------------------------------------------------------------------------------- 1 | # a general parser for vmtrace 2 | import struct 3 | import sys 4 | import os 5 | import time 6 | import json 7 | import gc 8 | 9 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 10 | sys.path.append(os.path.dirname(SCRIPT_DIR)) 11 | 12 | from parserPackage.decoder import decoder 13 | from fetchPackage.fetchTrace import fetcher 14 | from utilsPackage.compressor import writeCompressedJson, readCompressedJson 15 | 16 | 17 | 18 | class TxSpectorTranslator: 19 | def __init__(self): 20 | self.decoder = decoder() 21 | 22 | 23 | def parseLogs(self, trace): 24 | structLogs = trace['structLogs'] 25 | self.callReserved = {} 26 | # sometimes we need to rely on last return to determine this call's return value 27 | lastReturn = (0, 0) 28 | 29 | translated = "" 30 | ii = -1 31 | ii_handled = -1 32 | while ii < len(structLogs) - 1: 33 | ii += 1 34 | pc = structLogs[ii]["pc"] 35 | opcode = structLogs[ii]["op"] 36 | if opcode == "KECCAK256": 37 | opcode = "SHA3" 38 | 39 | if ii != 0 and ii != ii_handled: 40 | lastDepth = structLogs[ii-1]["depth"] 41 | depth = structLogs[ii]["depth"] 42 | if lastDepth > depth: 43 | if depth in self.callReserved: 44 | callResultHex = structLogs[ii]["stack"][-1] 45 | callResult = int(callResultHex, 16) 46 | toAdd, opcode, retOffset, retLength = self.callReserved[depth] 47 | 48 | if opcode == "CALL" or opcode == "STATICCALL" or opcode == "CALLCODE" or opcode == "DELEGATECALL": 49 | retValueHex = self.decoder.extractMemory(structLogs[ii]["memory"], retOffset, retLength) 50 | retValue = 0 51 | if retValueHex != "": 52 | retValue = int(retValueHex, 16) 53 | if retValue == 0 and lastReturn[0] == lastDepth and structLogs[ii-1]["op"] == "RETURN": 54 | retValue = lastReturn[1] 55 | 56 | toAdd = "{}{},{}\n".format(toAdd, callResult, retValue) 57 | translated += toAdd 58 | ii_handled = ii 59 | # print("Use Reserved at pc = {}, ii = {}, lastDepth = {}, depth = {}, callResult = {}, retValue = {}".format(pc, ii, lastDepth, depth, callResult, retValue)) 60 | del self.callReserved[depth] 61 | ii -= 1 62 | continue 63 | 64 | elif opcode == "CREATE" or opcode == "CREATE2": 65 | retValue = structLogs[ii]["stack"][-1] 66 | toAdd = "{}{}\n".format(toAdd, retValue) 67 | translated += toAdd 68 | ii_handled = ii 69 | # print("Use Reserved at pc = {}, ii = {}, lastDepth = {}, depth = {}, retValue = {}".format(pc, ii, lastDepth, depth, retValue)) 70 | del self.callReserved[depth] 71 | ii -= 1 72 | continue 73 | 74 | 75 | else: 76 | sys.exit("Error: depth mismatch, pc = {}, ii = {}, lastDepth = {}, depth = {}".format(pc, ii, lastDepth, depth)) 77 | 78 | 79 | toAdd = "{};{};".format(pc, opcode) 80 | if opcode == "ADD" or opcode == "MUL" or opcode == "SUB" or \ 81 | opcode == "DIV" or opcode == "SDIV" or opcode == "MOD" or opcode == "SMOD" or \ 82 | opcode == "ADDMOD" or opcode == "MULMOD" or opcode == "EXP" or opcode == "SIGNEXTEND" or \ 83 | opcode == "LT" or opcode == "GT" or opcode == "SLT" or opcode == "SGT" or \ 84 | opcode == "EQ" or opcode == "ISZERO" or opcode == "AND" or opcode == "OR" or \ 85 | opcode == "XOR" or opcode == "NOT": 86 | pass# confirmed 87 | elif opcode == "BYTE": 88 | pass 89 | elif opcode == "SHL": 90 | pass 91 | elif opcode == "SHR": 92 | pass 93 | elif opcode == "SAR": 94 | pass 95 | elif opcode == "SHA3" or \ 96 | opcode == "ADDRESS" or opcode == "BALANCE" or opcode == "ORIGIN" or \ 97 | opcode == "CALLER" or opcode == "CALLVALUE" or opcode == "CALLDATALOAD" or \ 98 | opcode == "CALLDATASIZE" or opcode == "CODESIZE" or opcode == "GASPRICE" or opcode == "TXGASPRICE" or \ 99 | opcode == "EXTCODESIZE" or opcode == "RETURNDATASIZE" or opcode == "EXTCODEHASH" or \ 100 | opcode == "BLOCKHASH" or opcode == "COINBASE" or opcode == "TIMESTAMP" or \ 101 | opcode == "NUMBER" or opcode == "DIFFICULTY" or opcode == "GASLIMIT" or \ 102 | opcode == "CHAINID" or opcode == "SELFBALANCE" or opcode == "BASEFEE" or \ 103 | opcode == "MLOAD" or opcode == "SLOAD" or opcode == "PC" or \ 104 | opcode == "MSIZE" or opcode == "GAS": 105 | valueHex = structLogs[ii+1]["stack"][-1] 106 | value = int(valueHex, 16) 107 | toAdd += "{}".format(value) 108 | 109 | 110 | elif opcode == "CALLDATACOPY" or opcode == "CODECOPY" or opcode == "RETURNDATACOPY": 111 | structLog = structLogs[ii] 112 | nextStructLog = structLogs[ii+1] 113 | destOffset = structLog["stack"][-1] 114 | offset = structLog["stack"][-2] 115 | length = structLog["stack"][-3] 116 | destOffsetInt = int(destOffset, base = 16) 117 | offsetInt = int(offset, base = 16) 118 | lengthInt = int(length, base = 16) 119 | valueHex = self.decoder.extractMemory(nextStructLog["memory"], destOffset, length) 120 | value = 0 121 | if valueHex != "": 122 | value = int(valueHex, 16) 123 | 124 | toAdd += "{}".format(value) 125 | 126 | elif opcode == "EXTCODECOPY": 127 | structLog = structLogs[ii] 128 | nextStructLog = structLogs[ii+1] 129 | address = structLog["stack"][-1] 130 | destOffset = structLog["stack"][-2] 131 | offset = structLog["stack"][-3] 132 | length = structLog["stack"][-4] 133 | destOffsetInt = int(destOffset, base = 16) 134 | offsetInt = int(offset, base = 16) 135 | lengthInt = int(length, base = 16) 136 | valueHex = self.decoder.extractMemory(nextStructLog["memory"], destOffset, length) 137 | value = 0 138 | if valueHex != "": 139 | value = int(valueHex, 16) 140 | toAdd += "{}".format(value) 141 | 142 | elif opcode == "POP": 143 | pass# confirmed 144 | elif opcode == "MSTORE": 145 | pass # confirmed 146 | elif opcode == "MSTORE8": 147 | pass# confirmed 148 | elif opcode == "SSTORE": 149 | pass# confirmed 150 | elif opcode == "JUMP": 151 | pass# confirmed 152 | elif opcode == "JUMPI": 153 | pass# confirmed 154 | elif opcode == "JUMPDEST": 155 | pass# confirmed 156 | elif opcode.startswith("PUSH"): # PUSH0-PUSH32 157 | valueHex = structLogs[ii+1]["stack"][-1] 158 | value = int(valueHex, 16) 159 | toAdd += "{}".format(value) 160 | elif opcode.startswith("DUP"): # DUP1-DUP16 161 | pass # confirmed 162 | elif opcode.startswith("SWAP"): 163 | pass # confirmed 164 | elif opcode.startswith("LOG"): 165 | pass # confirmed 166 | 167 | elif opcode == "CREATE": 168 | structLog = structLogs[ii] 169 | nextStructLog = structLogs[ii+1] 170 | 171 | # valueHex = nextStructLog["stack"][-1] 172 | # offsetHex = nextStructLog["stack"][-2] 173 | # lengthHex = nextStructLog["stack"][-3] 174 | 175 | depth = structLog["depth"] 176 | nextDepth = nextStructLog["depth"] 177 | 178 | # print("Reserve create at depth {}: toAdd-{}".format(depth, toAdd)) 179 | 180 | self.callReserved[depth] = (toAdd, opcode, None, None) 181 | continue 182 | 183 | elif opcode == "CREATE2": 184 | structLog = structLogs[ii] 185 | nextStructLog = structLogs[ii+1] 186 | 187 | # valueHex = nextStructLog["stack"][-1] 188 | # offsetHex = nextStructLog["stack"][-2] 189 | # lengthHex = nextStructLog["stack"][-3] 190 | 191 | depth = structLog["depth"] 192 | nextDepth = nextStructLog["depth"] 193 | 194 | # print("Reserve create2 at depth {}: toAdd-{}".format(depth, toAdd)) 195 | 196 | self.callReserved[depth] = (toAdd, opcode, None, None) 197 | continue 198 | 199 | 200 | # Four call opcodes has a special type: 0,1, they need extra type 201 | # value_extra is used to store more arguments for call, callcode, delegatecall, staticcall 202 | # op.value is success flag, value_extra is the memory content. 203 | 204 | 205 | elif opcode == "CALL": 206 | structLog = structLogs[ii] 207 | nextStructLog = structLogs[ii+1] 208 | 209 | gas = structLog["stack"][-1] 210 | addr = structLog["stack"][-2] 211 | value = structLog["stack"][-3] 212 | argsOffset = structLog["stack"][-4] 213 | argsLength = structLog["stack"][-5] 214 | retOffset = structLog["stack"][-6] 215 | retLength = structLog["stack"][-7] 216 | 217 | depth = structLog["depth"] 218 | nextDepth = nextStructLog["depth"] 219 | # precompile 220 | if depth == nextDepth: 221 | successValueHex = nextStructLog["stack"][-1] 222 | successValue = int(successValueHex, 16) 223 | retValueHex = self.decoder.extractMemory(nextStructLog["memory"], retOffset, retLength) 224 | retValue = 0 225 | if retValueHex != "": 226 | retValue = int(retValueHex, 16) 227 | toAdd += "{},{}".format(successValue, retValue) 228 | else: 229 | # print("Reserve call at depth {}: toAdd-{}, retOffset-{}, retLength-{}".format(depth, toAdd, retOffset, retLength)) 230 | if depth in self.callReserved: 231 | print("Error: depth {} is already reserved".format(depth)) 232 | self.callReserved[depth] = (toAdd, opcode, retOffset, retLength) 233 | continue 234 | 235 | 236 | elif opcode == "CALLCODE": 237 | 238 | structLog = structLogs[ii] 239 | nextStructLog = structLogs[ii+1] 240 | 241 | gas = structLog["stack"][-1] 242 | addr = structLog["stack"][-2] 243 | value = structLog["stack"][-3] 244 | argsOffset = structLog["stack"][-4] 245 | argsLength = structLog["stack"][-5] 246 | retOffset = structLog["stack"][-6] 247 | retLength = structLog["stack"][-7] 248 | 249 | depth = structLog["depth"] 250 | nextDepth = nextStructLog["depth"] 251 | # precompile 252 | if depth == nextDepth: 253 | successValueHex = nextStructLog["stack"][-1] 254 | successValue = int(successValueHex, 16) 255 | retValueHex = self.decoder.extractMemory(nextStructLog["memory"], retOffset, retLength) 256 | retValue = 0 257 | if retValueHex != "": 258 | retValue = int(retValueHex, 16) 259 | toAdd += "{},{}".format(successValue, retValue) 260 | else: 261 | # print("Reserve call at depth {}: toAdd-{}, retOffset-{}, retLength-{}".format(depth, toAdd, retOffset, retLength)) 262 | if depth in self.callReserved: 263 | print("Error: depth {} is already reserved".format(depth)) 264 | self.callReserved[depth] = (toAdd, opcode, retOffset, retLength) 265 | continue 266 | 267 | 268 | elif opcode == "STATICCALL" or opcode == "DELEGATECALL": 269 | structLog = structLogs[ii] 270 | nextStructLog = structLogs[ii+1] 271 | 272 | gas = structLog["stack"][-1] 273 | addr = structLog["stack"][-2] 274 | argsOffset = structLog["stack"][-3] 275 | argsLength = structLog["stack"][-4] 276 | retOffset = structLog["stack"][-5] 277 | retLength = structLog["stack"][-6] 278 | 279 | depth = structLog["depth"] 280 | nextDepth = nextStructLog["depth"] 281 | 282 | # precompile 283 | if depth == nextDepth: 284 | successValueHex = nextStructLog["stack"][-1] 285 | successValue = int(successValueHex, 16) 286 | retValueHex = self.decoder.extractMemory(nextStructLog["memory"], retOffset, retLength) 287 | retValue = 0 288 | if retValueHex != "": 289 | retValue = int(retValueHex, 16) 290 | toAdd += "{},{}".format(successValue, retValue) 291 | else: 292 | # print("Reserve call at depth {}: toAdd-{}, retOffset-{}, retLength-{}".format(depth, toAdd, retOffset, retLength)) 293 | if depth in self.callReserved: 294 | print("Error: depth {} is already reserved".format(depth)) 295 | self.callReserved[depth] = (toAdd, opcode, retOffset, retLength) 296 | continue 297 | 298 | 299 | 300 | elif opcode == "STOP": 301 | pass # confirmed 302 | elif opcode == "RETURN" or opcode == "REVERT": 303 | offset = structLogs[ii]["stack"][-1] 304 | length = structLogs[ii]["stack"][-2] 305 | offsetInt = int(offset, 16) 306 | lengthInt = int(length, 16) 307 | 308 | valueHex = self.decoder.extractMemory(structLogs[ii]["memory"], offset, length) 309 | value = 0 310 | if valueHex != "": 311 | value = int(valueHex, 16) 312 | 313 | toAdd += "{}".format(value) 314 | 315 | depth = structLogs[ii]["depth"] 316 | lastReturn = (depth, value) 317 | 318 | 319 | elif opcode == "SELFDESTRUCT": 320 | pass 321 | else: 322 | sys.exit("Error: unknown opcode {}".format(opcode)) 323 | 324 | translated += toAdd + "\n" 325 | return translated 326 | 327 | 328 | 329 | 330 | 331 | 332 | def solve1benchmark(exploitTx, use_cache = True): 333 | 334 | fe = fetcher() 335 | result_dict = None 336 | # path = SCRIPT_DIR + '/../TxSpectorHelper/cache2/{}.json'.format(exploitTx) 337 | # if not (use_cache and os.path.exists(path)): 338 | # result_dict = fe.getTrace(exploitTx, FullTrace=False) 339 | # with open(path, 'w') as f: 340 | # json.dump(result_dict, f, indent = 2) 341 | 342 | 343 | path = SCRIPT_DIR + '/../TxSpectorHelper/cache/{}.json.gz'.format(exploitTx) 344 | if not (use_cache and os.path.exists(path)): 345 | result_dict = fe.getTrace(exploitTx, FullTrace=False, FullStack=True) 346 | # with open(path, 'w') as f: 347 | # json.dump(result_dict, f, indent = 2) 348 | writeCompressedJson(path, result_dict) 349 | 350 | path2 = SCRIPT_DIR + '/../TxSpectorHelper/example.json' 351 | 352 | for ii, op in enumerate(result_dict["structLogs"]): 353 | result_dict["structLogs"][ii]["len"] = len(op["stack"]) 354 | 355 | # store the trace in example.json 356 | with open(path2, 'w') as f: 357 | json.dump(result_dict, f, indent = 2) 358 | 359 | 360 | kk = readCompressedJson(path) 361 | translated = TxSpectorTranslator().parseLogs(kk) 362 | path = SCRIPT_DIR + '/../TxSpectorHelper/translated/{}.txt'.format(exploitTx) 363 | with open("{}.txt".format(exploitTx), "w") as f: 364 | f.write(translated) 365 | 366 | gc.collect() 367 | 368 | 369 | 370 | if __name__ == "__main__": 371 | 372 | solve1benchmark("0xadbc02bda46eb54e411ca73655c9b6993805c75535be3084ea316cd334b35c9c", False) -------------------------------------------------------------------------------- /constraintPackage/README.md: -------------------------------------------------------------------------------- 1 | ### Constraint Learning Engine 2 | 3 | The learning engine starts with a set of benign training inputs. 4 | 5 | It infers safety constraints from the training inputs. 6 | 7 | Safety constraints define the comfort zone of the application. 8 | 9 | 10 | -------------------------------------------------------------------------------- /constraintPackage/accessControlInfer.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | from parserPackage.locator import * 4 | from parserPackage.parser import proxyMap 5 | from trackerPackage.dataSource import * 6 | from crawlPackage.crawlQuicknode import CrawlQuickNode 7 | from crawlPackage.crawlEtherscan import CrawlEtherscan 8 | from utilsPackage.compressor import * 9 | from staticAnalyzer.analyzer import Analyzer 10 | 11 | 12 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 13 | sys.path.append(os.path.dirname(SCRIPT_DIR)) 14 | 15 | import toml 16 | settings = toml.load("settings.toml") 17 | 18 | 19 | # check 3 invariants: 20 | # 1. msg.sender != tx.origin 21 | # 2. is msg.sender/tx.origin owner? 22 | # 3. is msg.sender/tx.origin manager? 23 | def inferAccessControl(accesslistTable): 24 | crawlQuickNode = CrawlQuickNode() 25 | crawlEtherscan = CrawlEtherscan() 26 | 27 | for contract, accessList in accesslistTable: 28 | if contract in proxyMap: 29 | contract = proxyMap[contract] 30 | # build read-only functions 31 | ABI = crawlEtherscan.Contract2ABI(contract) 32 | readOnlyFunctions = ["fallback"] 33 | nonReadOnlyFunctions = [] 34 | for function in ABI: 35 | if function["type"] == "function" and (function["stateMutability"] == "view" or function["stateMutability"] == "pure"): 36 | readOnlyFunctions.append(function["name"]) 37 | if function["type"] == "function" and (function["stateMutability"] != "view" and function["stateMutability"] != "pure"): 38 | nonReadOnlyFunctions.append(function["name"]) 39 | 40 | 41 | # stage 1: training 42 | accessControlMap = {} 43 | # { "func": 44 | # { 45 | # "origin!=sender": 0, 46 | # "sender": { 47 | # addr: times, 48 | # } 49 | # "origin": { 50 | # addr: times, 51 | # } 52 | # } 53 | # } 54 | invariantMap = {} 55 | # { "func": 56 | # { 57 | # "require(origin==sender)": true, 58 | # "isSenderOwner": addr, 59 | # "isSenderManager": a list of addr, 60 | # "isOriginOwner": addr, 61 | # "isOriginManager": a list of addr, 62 | # } 63 | # } 64 | counter = -1 65 | analyzer = Analyzer() 66 | for tx, funcCallList in accessList: 67 | counter += 1 68 | origin = crawlQuickNode.Tx2Details(tx)["from"].lower() 69 | if len(funcCallList) != 1: 70 | print(funcCallList) 71 | sys.exit("access control infer: not one function call in a transaction") 72 | 73 | for funcCall in funcCallList[0]: 74 | sender = funcCall["msg.sender"].lower() 75 | name = "" 76 | if "type" in funcCall and funcCall["type"] == "staticcall": 77 | continue 78 | 79 | if "name" in funcCall: 80 | if funcCall["name"] == "fallback" and funcCall["Selector"] != "0x": 81 | # get real funcName from selector and contract 82 | funcSigMap = analyzer.contract2funcSigMap(contract) 83 | funcCall["name"] = funcSigMap[funcCall["Selector"].lower()][0] 84 | name += funcCall["name"] 85 | if funcCall["name"] not in nonReadOnlyFunctions: 86 | continue 87 | 88 | if name == "": 89 | if "structLogsStart" in funcCall and "structLogsEnd" in funcCall and \ 90 | funcCall["structLogsEnd"] - funcCall["structLogsStart"] <= 20 and \ 91 | "type" in funcCall and funcCall["type"] == "delegatecall": 92 | continue 93 | else: 94 | print(tx) 95 | sys.exit("access control infer: name is empty") 96 | pass 97 | 98 | if "Selector" in funcCall: 99 | name += funcCall["Selector"] 100 | 101 | if name not in accessControlMap: 102 | c = int(origin != sender) 103 | accessControlMap[name] = { 104 | "origin!=sender": c, 105 | "sender": {sender: 1}, 106 | "origin": {origin: 1} 107 | } 108 | else: 109 | if origin != sender: 110 | accessControlMap[name]["origin!=sender"] += 1 111 | # print("func", name, "origin", origin, "sender", sender) 112 | 113 | 114 | if sender not in accessControlMap[name]["sender"]: 115 | accessControlMap[name]["sender"][sender] = 1 116 | else: 117 | accessControlMap[name]["sender"][sender] += 1 118 | 119 | if origin not in accessControlMap[name]["origin"]: 120 | accessControlMap[name]["origin"][origin] = 1 121 | else: 122 | accessControlMap[name]["origin"][origin] += 1 123 | 124 | 125 | 126 | # stage 2: infer 127 | for func in accessControlMap: 128 | invariantMap[func] = {} 129 | # invariant 1: origin != sender 130 | if accessControlMap[func]["origin!=sender"] == 0: 131 | invariantMap[func]["require(origin==sender)"] = True 132 | else: 133 | invariantMap[func]["require(origin==sender)"] = False 134 | 135 | # invariant 2: sender is owner or manager, origin is owner or manager 136 | if len(accessControlMap[func]["sender"].keys()) == 1: 137 | invariantMap[func]["isSenderOwner"] = list(accessControlMap[func]["sender"].keys())[0] 138 | if len(accessControlMap[func]["sender"].keys()) >= 5: 139 | print("func {} has more than 5 senders".format(func)) 140 | elif len(accessControlMap[func]["sender"].keys()) == 1: 141 | print("func {} has only 1 sender".format(func)) 142 | else: 143 | invariantMap[func]["isSenderManager"] = list(accessControlMap[func]["sender"].keys()) 144 | 145 | if len(accessControlMap[func]["origin"].keys()) == 1: 146 | invariantMap[func]["isOriginOwner"] = list(accessControlMap[func]["origin"].keys())[0] 147 | if len(accessControlMap[func]["origin"].keys()) >= 5: 148 | print("func {} has more than 5 origins".format(func)) 149 | elif len(accessControlMap[func]["origin"].keys()) == 1: 150 | print("func {} has only 1 origin".format(func)) 151 | else: 152 | invariantMap[func]["isOriginManager"] = list(accessControlMap[func]["origin"].keys()) 153 | 154 | 155 | print("==invariant map: ") 156 | import pprint 157 | pp = pprint.PrettyPrinter(indent=2) 158 | pp.pprint(invariantMap) 159 | 160 | 161 | print("Interpretation of the above invariant map: ") 162 | 163 | for func in invariantMap: 164 | print("For the function {}:".format(func)) 165 | for key in invariantMap[func]: 166 | if key == "isSenderManager" or key == "isOriginManager": 167 | print("\tthe invariant {} has parameters {}".format(key, invariantMap[func][key])) 168 | else: 169 | print("\tis the invariant {} satisfied? {}".format(key, invariantMap[func][key])) 170 | -------------------------------------------------------------------------------- /constraintPackage/gasControlInfer.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | from parserPackage.locator import * 4 | from parserPackage.parser import proxyMap 5 | import copy 6 | from trackerPackage.dataSource import * 7 | from crawlPackage.crawlEtherscan import CrawlEtherscan 8 | from utilsPackage.compressor import * 9 | from staticAnalyzer.analyzer import Analyzer 10 | 11 | 12 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 13 | sys.path.append(os.path.dirname(SCRIPT_DIR)) 14 | import toml 15 | 16 | 17 | 18 | def inferGasControl(accesslistTable): 19 | crawlEtherscan = CrawlEtherscan() 20 | 21 | for contract, accessList in accesslistTable: 22 | if contract in proxyMap: 23 | contract = proxyMap[contract] 24 | # build read-only functions 25 | ABI = crawlEtherscan.Contract2ABI(contract) 26 | readOnlyFunctions = ["fallback"] 27 | nonReadOnlyFunctions = [] 28 | for function in ABI: 29 | if function["type"] == "function" and (function["stateMutability"] == "view" or function["stateMutability"] == "pure"): 30 | readOnlyFunctions.append(function["name"]) 31 | if function["type"] == "function" and (function["stateMutability"] != "view" and function["stateMutability"] != "pure"): 32 | nonReadOnlyFunctions.append(function["name"]) 33 | 34 | 35 | # stage 1: training 36 | gasControlMap = {} 37 | # { "func": 38 | # { 39 | # gasStart: [gasStart1, gasStart2, ...], 40 | # gasEnd: [gasEnd1, gasEnd2, ...], 41 | # } 42 | 43 | counter = -1 44 | analyzer = Analyzer() 45 | for tx, funcCallList in accessList: 46 | counter += 1 47 | if len(funcCallList) != 1: 48 | print(funcCallList) 49 | sys.exit("gas control infer: not one function call in a transaction") 50 | 51 | for funcCall in funcCallList[0]: 52 | name = "" 53 | if "name" in funcCall: 54 | if funcCall["name"] == "fallback" and funcCall["Selector"] != "0x": 55 | # get real funcName from selector and contract 56 | funcSigMap = analyzer.contract2funcSigMap(contract) 57 | funcCall["name"] = funcSigMap[funcCall["Selector"].lower()][0] 58 | 59 | name += funcCall["name"] 60 | if funcCall["name"] not in nonReadOnlyFunctions: 61 | continue 62 | 63 | if "Selector" in funcCall: 64 | name += funcCall["Selector"] 65 | 66 | if "gas" not in funcCall: 67 | print("no gas") 68 | print(tx) 69 | continue 70 | gasStart = funcCall["gas"] 71 | if "gasEnd" not in funcCall: 72 | print("no gasEnd") 73 | print(tx) 74 | continue 75 | gasEnd = funcCall["gasEnd"] 76 | 77 | if name not in gasControlMap: 78 | gasControlMap[name] = { 79 | "gasStart": [gasStart], 80 | "gasEnd": [gasEnd], 81 | } 82 | else: 83 | gasControlMap[name]["gasStart"].append(gasStart) 84 | gasControlMap[name]["gasEnd"].append(gasEnd) 85 | 86 | 87 | 88 | # stage 2: infer 89 | invariantMap = {} 90 | # { "func": 91 | # { 92 | # "require(gasStart <= constant)": b, 93 | # "require(gasStart - gasEnd <= constant)": b, 94 | # } 95 | # } 96 | for func in gasControlMap: 97 | invariantMap[func] = {} 98 | # invariant 1: require gasStart < constant 99 | maxValue = max(gasControlMap[func]["gasStart"]) 100 | minValue = min(gasControlMap[func]["gasStart"]) 101 | 102 | if len(gasControlMap[func]["gasStart"]) >= 2 and maxValue != minValue: 103 | invariantMap[func]["require(gasStart <= constant)"] = maxValue 104 | 105 | # invariant 2: require gasStart - gasEnd < constant 106 | # and invariant 3: require gasStart - gasEnd > constant 107 | result = [a - b for a, b in zip(gasControlMap[func]["gasStart"], gasControlMap[func]["gasEnd"])] 108 | 109 | maxGapValue = max( result ) 110 | minGapValue = min( result ) 111 | 112 | if len(gasControlMap[func]["gasEnd"]) >= 2 and maxGapValue != minGapValue: 113 | invariantMap[func]["require(gasStart - gasEnd <= constant)"] = maxGapValue 114 | 115 | 116 | isHavingGasConsumed = False 117 | print("==invariant map: ") 118 | import pprint 119 | pp = pprint.PrettyPrinter(indent=2) 120 | pp.pprint(invariantMap) 121 | 122 | print("Interpretation of the above invariant map: ") 123 | for func in invariantMap: 124 | if "require(gasStart <= constant)" in invariantMap[func]: 125 | print("\tfunction {} requires gasStart <= {}".format(func, invariantMap[func]["require(gasStart <= constant)"])) 126 | if "require(gasStart - gasEnd <= constant)" in invariantMap[func]: 127 | print("\tfunction {} requires gasStart - gasEnd <= {}".format(func, invariantMap[func]["require(gasStart - gasEnd <= constant)"])) 128 | -------------------------------------------------------------------------------- /constraintPackage/moneyFlowInfer.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | from parserPackage.locator import * 4 | from parserPackage.parser import proxyMap 5 | from trackerPackage.dataSource import * 6 | from crawlPackage.crawlQuicknode import CrawlQuickNode 7 | from crawlPackage.crawlEtherscan import CrawlEtherscan 8 | from utilsPackage.compressor import * 9 | from parserPackage.decoder import * 10 | 11 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 12 | sys.path.append(os.path.dirname(SCRIPT_DIR)) 13 | 14 | import toml 15 | settings = toml.load("settings.toml") 16 | 17 | import numpy as np 18 | 19 | 20 | 21 | 22 | ["0x6c3f90f043a72fa612cbac8115ee7e52bde6e490", # 3cRV good! 2 sstores transferFrom 0, 1 23 | "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", # USDC good! 2 sstores 24 | "0xdac17f958d2ee523a2206206994597c13d831ec7", # USDT good! 2 sstores 25 | "0x853d955acef822db058eb8505911ed77f175b99e", # Frax good! 2 sstores 26 | "0x6b175474e89094c44da98b954eedeac495271d0f", # DAI good! 2 sstores 27 | "0x865377367054516e17014ccded1e7d814edc9ce4", # DOLA good! 2 sstores 28 | "0xff20817765cb7f73d4bde2e66e067e58d11095c2", # AMP CreamFi1_1 29 | "0x1f9840a85d5af5bf1d1762f925bdaddc4201f984", # UNI good! 2 sstores 30 | "0x956f47f50a910163d8bf957cf5846d573e7f87ca", # FEI good! 2 sstores 31 | "0xf938424f7210f31df2aee3011291b658f872e91e", # Visor good! 2 sstores 32 | "0xb1bbeea2da2905e6b0a30203aef55c399c53d042", # Uniswap UMB4 good! 2 sstores 33 | "0x56de8bc61346321d4f2211e3ac3c0a7f00db9b76"] # Rena good! 2 sstores 34 | 35 | benchmark2token = { 36 | "BeanstalkFarms": "0x6c3f90f043a72fa612cbac8115ee7e52bde6e490", 37 | "BeanstalkFarms_interface": None, 38 | "HarmonyBridge": "ether", 39 | "HarmonyBridge_interface": "ether", 40 | "XCarnival": "ether", 41 | "RariCapital2_1": "ether", 42 | "RariCapital2_2": "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", 43 | "RariCapital2_3": "0xdac17f958d2ee523a2206206994597c13d831ec7", 44 | "RariCapital2_4": "0x853d955acef822db058eb8505911ed77f175b99e", 45 | "DODO": "0xdac17f958d2ee523a2206206994597c13d831ec7", 46 | "PickleFi": "0x6b175474e89094c44da98b954eedeac495271d0f", 47 | "Nomad": "0x2260fac5e5542a773aa44fbcfedf7c193bc2c599", 48 | "PolyNetwork": "ether", 49 | "Punk_1": "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", 50 | "Punk_2": "0xdac17f958d2ee523a2206206994597c13d831ec7", 51 | "Punk_3": "0x6b175474e89094c44da98b954eedeac495271d0f", 52 | "SaddleFi": "0x6b175474e89094c44da98b954eedeac495271d0f", 53 | "Eminence": "0x6b175474e89094c44da98b954eedeac495271d0f", 54 | "Harvest1_fUSDT": "0xdac17f958d2ee523a2206206994597c13d831ec7", 55 | "Harvest2_fUSDC": "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", 56 | "bZx2": "ether", 57 | "Warp": "0x6b175474e89094c44da98b954eedeac495271d0f", 58 | "Warp_interface": "0x6b175474e89094c44da98b954eedeac495271d0f", 59 | "CheeseBank_1": "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", 60 | "CheeseBank_2": "0xdac17f958d2ee523a2206206994597c13d831ec7", 61 | "CheeseBank_3": "0x6b175474e89094c44da98b954eedeac495271d0f", 62 | "ValueDeFi": "0x6c3f90f043a72fa612cbac8115ee7e52bde6e490", 63 | "InverseFi": "0x865377367054516e17014ccded1e7d814edc9ce4", 64 | "Yearn1": "0x6b175474e89094c44da98b954eedeac495271d0f", 65 | "Yearn1_interface": "0x6b175474e89094c44da98b954eedeac495271d0f", 66 | "Opyn": "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", 67 | "CreamFi1_1": "0xff20817765cb7f73d4bde2e66e067e58d11095c2", 68 | "CreamFi1_2": "ether", 69 | "IndexFi": "0x1f9840a85d5af5bf1d1762f925bdaddc4201f984", 70 | "CreamFi2_1": "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", 71 | "CreamFi2_2": "0xdac17f958d2ee523a2206206994597c13d831ec7", 72 | "CreamFi2_3": "0x1f9840a85d5af5bf1d1762f925bdaddc4201f984", 73 | "CreamFi2_4": "0x956f47f50a910163d8bf957cf5846d573e7f87ca", 74 | "RariCapital1": "ether", 75 | "VisorFi": "0xf938424f7210f31df2aee3011291b658f872e91e", 76 | "UmbrellaNetwork": "0xb1bbeea2da2905e6b0a30203aef55c399c53d042", 77 | "RevestFi": "0x56de8bc61346321d4f2211e3ac3c0a7f00db9b76", 78 | "RevestFi_interface": None, 79 | "RoninNetwork": "ether" 80 | 81 | } 82 | 83 | 84 | benchmark2vault = { 85 | "BeanstalkFarms": "0x3a70dfa7d2262988064a2d051dd47521e43c9bdd", 86 | "BeanstalkFarms_interface": "0xc1e088fc1323b20bcbee9bd1b9fc9546db5624c5", 87 | "HarmonyBridge": "0xf9fb1c508ff49f78b60d3a96dea99fa5d7f3a8a6", 88 | "HarmonyBridge_interface": "0x715cdda5e9ad30a0ced14940f9997ee611496de6", 89 | "XCarnival": "0xb38707e31c813f832ef71c70731ed80b45b85b2d", # 0x5417da20ac8157dd5c07230cfc2b226fdcfc5663", 90 | "RariCapital2_1": "0x26267e41ceca7c8e0f143554af707336f27fa051", 91 | "RariCapital2_2": "0xebe0d1cb6a0b8569929e062d67bfbc07608f0a47", 92 | "RariCapital2_3": "0xe097783483d1b7527152ef8b150b99b9b2700c8d", 93 | "RariCapital2_4": "0x8922c1147e141c055fddfc0ed5a119f3378c8ef8", 94 | "DODO": "0x051ebd717311350f1684f89335bed4abd083a2b6", # "0x509ef8c68e7d246aab686b6d9929998282a941fb", # "0x2bbd66fc4898242bdbd2583bbe1d76e8b8f71445", 95 | # "PickleFi": "0x6949bb624e8e8a90f87cd2058139fcd77d2f3f87", # "0x6847259b2B3A4c17e7c43C54409810aF48bA5210", 96 | "PickleFi": "0x6949bb624e8e8a90f87cd2058139fcd77d2f3f87", # "0x6847259b2B3A4c17e7c43C54409810aF48bA5210", 97 | 98 | "Nomad": "0x88a69b4e698a4b090df6cf5bd7b2d47325ad30a3", 99 | "PolyNetwork": "0x250e76987d838a75310c34bf422ea9f1ac4cc906", 100 | # "Punk_1": "0x3BC6aA2D25313ad794b2D67f83f21D341cc3f5fb", 101 | # "Punk_2": "0x1F3b04c8c96A31C7920372FFa95371C80A4bfb0D", 102 | # "Punk_3": "0x929cb86046E421abF7e1e02dE7836742654D49d6", 103 | "SaddleFi": "0x2069043d7556b1207a505eb459d18d908df29b55", 104 | 105 | "Eminence": "0x5ade7aE8660293F2ebfcEfaba91d141d72d221e8", 106 | "Harvest1_fUSDT": "0x053c80ea73dc6941f518a68e2fc52ac45bde7c9c", 107 | "Harvest2_fUSDC": "0xf0358e8c3cd5fa238a29301d0bea3d63a17bedbe", 108 | # "bZx2": "0x77f973fcaf871459aa58cd81881ce453759281bc", 109 | "Warp": "0x6046c3Ab74e6cE761d218B9117d5c63200f4b406", 110 | "Warp_interface": "0xba539b9a5c2d412cb10e5770435f362094f9541c", 111 | "CheeseBank_1": "0x5E181bDde2fA8af7265CB3124735E9a13779c021", 112 | "CheeseBank_2": "0x4c2a8A820940003cfE4a16294B239C8C55F29695", 113 | "CheeseBank_3": "0xA80e737Ded94E8D2483ec8d2E52892D9Eb94cF1f", 114 | # "ValueDeFi": "0x55bf8304c78ba6fe47fd251f37d7beb485f86d26", # "0xddd7df28b1fb668b77860b473af819b03db61101" 115 | "ValueDeFi": "0x55bf8304c78ba6fe47fd251f37d7beb485f86d26", # "0xddd7df28b1fb668b77860b473af819b03db61101" 116 | 117 | "InverseFi": "0x7Fcb7DAC61eE35b3D4a51117A7c58D53f0a8a670", 118 | # "Yearn1": "0x9c211bfa6dc329c5e757a223fb72f5481d676dc1", will not be used 119 | # "Yearn1": "0x9c211bfa6dc329c5e757a223fb72f5481d676dc1", will not be used 120 | 121 | "Yearn1_interface": "0xacd43e627e64355f1861cec6d3a6688b31a6f952", 122 | "Opyn": "0x951d51baefb72319d9fbe941e1615938d89abfe2", 123 | "CreamFi1_1": "0x2db6c82ce72c8d7d770ba1b5f5ed0b6e075066d6", # 0x3c710b981f5ef28da1807ce7ed3f2a28580e0754 124 | "CreamFi1_2": "0xd06527d5e56a3495252a528c4987003b712860ee", 125 | "IndexFi": "0xfa6de2697d59e88ed7fc4dfe5a33dac43565ea41", # "0x5bd628141c62a901e0a83e630ce5fafa95bbdee4", 126 | "CreamFi2_1": "0x44fbebd2f576670a6c33f6fc0b00aa8c5753b322", 127 | "CreamFi2_2": "0x797aab1ce7c01eb727ab980762ba88e7133d2157", 128 | "CreamFi2_3": "0xe89a6d0509faf730bd707bf868d9a2a744a363c7", 129 | "CreamFi2_4": "0x8c3b7a4320ba70f8239f83770c4015b5bc4e6f91", 130 | # "RariCapital1": "0xa422890cbbe5eaa8f1c88590fbab7f319d7e24b6", # 0xd6e194af3d9674b62d1b30ec676030c23961275e", # "0xec260f5a7a729bb3d0c42d292de159b4cb1844a3", 131 | "VisorFi": "0xc9f27a50f82571c1c8423a42970613b8dbda14ef", 132 | "UmbrellaNetwork": "0xb3fb1d01b07a706736ca175f827e4f56021b85de", 133 | "RevestFi": "0xa81bd16aa6f6b25e66965a2f842e9c806c0aa11f", 134 | "RevestFi_interface": "0x2320a28f52334d62622cc2eafa15de55f9987ed9", 135 | "RoninNetwork": "0x1a2a1c938ce3ec39b6d47113c7955baa9dd454f2", # 0x8407dc57739bcda7aa53ca6f12f82f9d51c2f21e 136 | 137 | } 138 | 139 | # the benchmark contracts that does not support money flow ratio analysis 140 | vaultContractBenchmarks = ["Punk_1", "Punk_2", "Punk_3", \ 141 | "ValueDeFi", "Yearn1", "RariCapital1", \ 142 | "bZx2", "PickleFi"] 143 | 144 | vaultContractBenchmarks = ["bZx2", "RariCapital1"] 145 | 146 | 147 | etherBenchmarks = [ 148 | "bZx2", "RoninNetwork", "HarmonyBridge", "XCarnival", \ 149 | 'RariCapital2_1', "RariCapital1", "PolyNetwork", \ 150 | "IndexFi" # UNI, its transfer and transferFrom haev multiple sstores 151 | ] 152 | 153 | 154 | # check X invariants 155 | # 1. upper bounds for data flow 156 | # 2. ranges for data flow 157 | 158 | def inferMoneyFlows(executionTable, originalContract, transferToken): 159 | crawlQuickNode = CrawlQuickNode() 160 | crawlEtherscan = CrawlEtherscan() 161 | 162 | 163 | for contract, executionList in executionTable: 164 | # build read-only functions 165 | ABI = crawlEtherscan.Contract2ABI(contract) 166 | readOnlyFunctions = ["fallback"] 167 | nonReadOnlyFunctions = [] 168 | for function in ABI: 169 | if function["type"] == "function" and (function["stateMutability"] == "view" or function["stateMutability"] == "pure"): 170 | readOnlyFunctions.append(function["name"]) 171 | if function["type"] == "function" and (function["stateMutability"] != "view" and function["stateMutability"] != "pure"): 172 | nonReadOnlyFunctions.append(function["name"]) 173 | 174 | # stage 1: training 175 | moneyFlowMap = { 176 | # { "func+type": 177 | # { 178 | # "transferAmount": [] 179 | # "transferPC": [] 180 | # "transferTokenBalance": [] 181 | # } 182 | # } 183 | } 184 | 185 | lastBlockBalance = [0, 0] 186 | blocks = [] 187 | 188 | for tx, dataS in executionList: 189 | if tx == "0x30fd944ddd5a68a9ab05048a243b852daf5f707e0448696b172cea89e757f4e5" or \ 190 | tx == "0x7ae864faf81979eba3bffa7a2a72f4ded858694ced79c63d613020d064bc06f4": 191 | # buggy tx 192 | continue 193 | 194 | 195 | sources = dataS["sources"] 196 | children = dataS["children"] 197 | metaData = dataS["metaData"] 198 | 199 | targetFunc = metaData["targetFunc"] 200 | targetFuncType = metaData["targetFuncType"] 201 | name = targetFunc + "+" + targetFuncType 202 | 203 | if name not in moneyFlowMap: 204 | moneyFlowMap[ name ] = { 205 | "transferAmount": [], 206 | "transferPC": [], 207 | } 208 | 209 | pc = None 210 | if "pc" in metaData: 211 | pc = metaData["pc"] 212 | elif len(sources) == 1 and isinstance(sources[0], str) and sources[0] == "msg.value": 213 | pc = -1 214 | else: 215 | sys.exit("dataFlowInfer: pc is not in metaData") 216 | if pc == None: 217 | sys.exit("dataFlowInfer: pc is None") 218 | # gas = metaData["gas"] 219 | # type = metaData["type"] 220 | # if type != "uint256": 221 | # print(type) 222 | 223 | transferAmount = None 224 | if "value" in metaData: 225 | transferAmount = metaData["value"] 226 | elif len(sources) == 1 and isinstance(sources[0], str) and sources[0] == "msg.value": 227 | # if "msg.value" not in metaData: 228 | # print(tx) 229 | # continue 230 | transferAmount = metaData["msg.value"] 231 | if isinstance(transferAmount, str): 232 | transferAmount = int(transferAmount, 16) 233 | else: 234 | print(sources) 235 | sys.exit("dataFlowInfer: transferAmount is not in metaData") 236 | 237 | if isinstance(transferAmount, str): 238 | transferAmount = int(transferAmount, 16) 239 | 240 | if transferAmount == None: 241 | print(sources) 242 | sys.exit("dataFlowInfer: transferAmount is None") 243 | 244 | 245 | moneyFlowMap[name]["transferAmount"].append(transferAmount) 246 | moneyFlowMap[name]["transferPC"].append(pc) 247 | 248 | transferTokenBalance = None 249 | block = crawlQuickNode.Tx2Block(tx) 250 | if "sstore" in metaData and len(metaData["sstore"]) == 3 and isinstance(metaData["sstore"][0], str) and \ 251 | (metaData["sstore"][0] == "transferFrom" or metaData["sstore"][0] == "transfer"): 252 | if metaData["sstore"][0] == "transferFrom": 253 | postBalance = int(metaData["sstore"][2][1], 16) 254 | transferTokenBalance = postBalance - transferAmount 255 | lastBlockBalance = [block, transferTokenBalance] 256 | elif metaData["sstore"][0] == "transfer": 257 | postBalance = int(metaData["sstore"][1][1], 16) 258 | transferTokenBalance = postBalance + transferAmount 259 | lastBlockBalance = [block, transferTokenBalance] 260 | else: 261 | # print("sstore not in metaData") 262 | if block not in blocks: 263 | blocks.append(block) 264 | blocks.append(block - 1) 265 | 266 | transferTokenBalance = None 267 | if lastBlockBalance[0] == block - 1: 268 | transferTokenBalance = lastBlockBalance[1] 269 | else: 270 | if lastBlockBalance[0] != 0: 271 | # we are about to replace lastBlockBalance 272 | temp = crawlQuickNode.TokenBalanceOf(transferToken, originalContract, lastBlockBalance[0] + 1) 273 | if temp != lastBlockBalance[1]: 274 | print("dataFlowInfer: error: temp={} != lastBlockBalance[1]={}".format(temp, lastBlockBalance[1])) 275 | sys.exit(0) 276 | transferTokenBalance = crawlQuickNode.TokenBalanceOf(transferToken, originalContract, block - 1) 277 | lastBlockBalance = [block - 1, transferTokenBalance] 278 | # print("lastBlockBalance = {}".format(lastBlockBalance)) 279 | 280 | if targetFuncType == "withdraw" or targetFuncType == "invest": 281 | lastBlockBalance[1] -= transferAmount 282 | if lastBlockBalance[1] < 0: 283 | sys.exit("dataFlowInfer: lastBlockBalance[1] < 0") 284 | elif targetFuncType == "deposit": 285 | lastBlockBalance[1] += transferAmount 286 | else: 287 | sys.exit("dataFlowInfer: targetFuncType is not withdraw or invest or deposit") 288 | 289 | if name in moneyFlowMap and "transferTokenBalance" in moneyFlowMap[name]: 290 | moneyFlowMap[name]["transferTokenBalance"].append(transferTokenBalance) 291 | 292 | 293 | for name in moneyFlowMap: 294 | if "transferPC" in moneyFlowMap[name] and len(moneyFlowMap[name]["transferPC"]) >= 2: 295 | maxPC = max(moneyFlowMap[name]["transferPC"]) 296 | minPC = min(moneyFlowMap[name]["transferPC"]) 297 | if maxPC != minPC and name != "unlock+withdraw" and name != "borrowTokenFromDeposit+deposit" and \ 298 | name != "deposit+deposit" and name != "redeemUnderlying+withdraw": 299 | print("name = {}, maxPC = {}, minPC = {}".format(name, maxPC, minPC)) 300 | # sys.exit("dataFlowInfer: error: maxPC != minPC") 301 | 302 | 303 | invariantMap = { 304 | # { "func": 305 | # { 306 | # "transferAmount": (smallest value, largest value) 307 | # "transferRatio": (smallest value, largest value) 308 | # } 309 | # } 310 | } 311 | 312 | # stage 2: infer 313 | # print(moneyFlowMap) 314 | for func in moneyFlowMap: 315 | maxValue = max(moneyFlowMap[func]["transferAmount"]) 316 | minValue = min(moneyFlowMap[func]["transferAmount"]) 317 | if len(moneyFlowMap[func]["transferAmount"]) >= 2 and maxValue != minValue: 318 | invariantMap[func] = {"transferAmount": (minValue, maxValue)} 319 | 320 | for func in moneyFlowMap: 321 | # transferRatio = transferAmount / transferTokenBalance 322 | if "transferAmount" in moneyFlowMap[func] and "transferTokenBalance" in moneyFlowMap[func]: 323 | transferRatioList = [] 324 | for x, y in zip(moneyFlowMap[func]["transferAmount"], moneyFlowMap[func]["transferTokenBalance"]): 325 | if y == 0 or x == 0: 326 | continue 327 | transferRatioList.append(x / y) 328 | # apply z-score method to filter outliers 329 | data = np.array(transferRatioList) 330 | data_mean = np.mean(data) 331 | data_std = np.std(data) 332 | transferRatioList = [x for x in transferRatioList if abs(x - data_mean) <= 3 * data_std] 333 | maxValue = max(transferRatioList) 334 | minValue = min(transferRatioList) 335 | if len(transferRatioList) >= 2 and maxValue != minValue: 336 | invariantMap[func]["transferRatio"] = (minValue, maxValue) 337 | 338 | print("==invariant map: ") 339 | print(invariantMap) 340 | print("Interpretation of the above invariant map: ") 341 | 342 | print("For the invariant moneyFlow:") 343 | 344 | for func in invariantMap: 345 | print("\tIt can be applied to function {}:".format(func)) 346 | print("\t with lowerbound of transferAmount = {}".format(invariantMap[func]["transferAmount"][0])) 347 | print("\t with upperbound of transferAmount = {}".format(invariantMap[func]["transferAmount"][1])) 348 | 349 | -------------------------------------------------------------------------------- /constraintPackage/reentrancyInfer.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | from parserPackage.locator import * 4 | import copy 5 | from trackerPackage.dataSource import * 6 | from fetchPackage.fetchTrace import fetcher 7 | from crawlPackage.crawlQuicknode import CrawlQuickNode 8 | from crawlPackage.crawlEtherscan import CrawlEtherscan 9 | from utilsPackage.compressor import * 10 | from staticAnalyzer.analyzer import Analyzer 11 | 12 | 13 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 14 | sys.path.append(os.path.dirname(SCRIPT_DIR)) 15 | 16 | import toml 17 | settings = toml.load("settings.toml") 18 | 19 | 20 | 21 | 22 | 23 | # check 2 invariants: 24 | # 1. re-entrant locks only on one type of functions - enter or exit 25 | # 2. re-entrant locks on both types of functions - enter and exit 26 | # 27 | # functions regarding token transfers cannot be re-entrant 28 | # for some special cases 29 | 30 | 31 | 32 | def inferReentrancy(accesslistTable, enterFunction, exitFunction): 33 | for contract, accessList in accesslistTable: 34 | tokenInFunctions = enterFunction 35 | tokenOutFunctions = exitFunction 36 | 37 | tokenMoveFunctions = tokenInFunctions + tokenOutFunctions 38 | tokenMoveFunctions = list(set(tokenMoveFunctions)) 39 | 40 | 41 | # check 1 invariant: 42 | # 1. re-entrant locks on both types of functions - enter and exit 43 | 44 | # functions regarding token transfers cannot be re-entrant 45 | # for some special cases 46 | # stage 1: training 47 | 48 | lastTokenMove = (0, []) 49 | # (transaction, [ (structLogsStart, structLogsEnd), ... ] ) 50 | 51 | invariantMap = { 52 | "NonReentrantLocks": len(tokenInFunctions) > 0 or len(tokenOutFunctions) > 0, 53 | } 54 | 55 | counter = -1 56 | analyzer = Analyzer() 57 | for tx, funcCallList in accessList: 58 | counter += 1 59 | if len(funcCallList) != 1: 60 | print(funcCallList) 61 | sys.exit("access control infer: not one function call in a transaction") 62 | 63 | for funcCall in funcCallList[0]: 64 | name = "" 65 | if "name" in funcCall: 66 | if funcCall["name"] == "fallback" and funcCall["Selector"] != "0x": 67 | # get real funcName from selector and contract 68 | funcSigMap = analyzer.contract2funcSigMap(contract) 69 | funcCall["name"] = funcSigMap[funcCall["Selector"].lower()][0] 70 | 71 | name += funcCall["name"] 72 | if funcCall["name"] not in tokenMoveFunctions: 73 | continue 74 | 75 | structLogsStart = funcCall["structLogsStart"] 76 | structLogsEnd = funcCall["structLogsEnd"] 77 | 78 | if name in tokenMoveFunctions: 79 | if tx == lastTokenMove[0]: 80 | for oldStructLogsStart, oldStructLogsEnd, oldName in lastTokenMove[1]: 81 | if (structLogsStart > oldStructLogsStart and structLogsEnd < oldStructLogsEnd) or \ 82 | (oldStructLogsStart > structLogsStart and oldStructLogsEnd < structLogsEnd): 83 | # find one re-entrancy 84 | invariantMap["NonReentrantLocks"] = False 85 | 86 | if structLogsStart == oldStructLogsStart and structLogsEnd == oldStructLogsEnd and name == oldName: 87 | # find one re-entrancy 88 | print("now is the time") 89 | lastTokenMove[1].append((structLogsStart, structLogsEnd, name)) 90 | else: 91 | lastTokenMove = (tx, [ (structLogsStart, structLogsEnd, name) ]) 92 | 93 | 94 | # stage 2: infer 95 | # functions regarding token transfers cannot be re-entrant 96 | # for some special cases 97 | print("==invariant list: ") 98 | print(invariantMap) 99 | 100 | print("Interpretation of the above invariant map: ") 101 | if invariantMap["NonReentrantLocks"] == False: 102 | print("re-entrancy guard cannot be applied") 103 | else: 104 | print("re-entrancy guard can be applied") 105 | 106 | -------------------------------------------------------------------------------- /constraintPackage/specialStorage.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | from parserPackage.locator import * 4 | from parserPackage.parser import proxyMap 5 | import copy 6 | from trackerPackage.dataSource import * 7 | from fetchPackage.fetchTrace import fetcher 8 | from crawlPackage.crawlQuicknode import CrawlQuickNode 9 | from crawlPackage.crawlEtherscan import CrawlEtherscan 10 | from utilsPackage.compressor import * 11 | 12 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 13 | sys.path.append(os.path.dirname(SCRIPT_DIR)) 14 | 15 | import toml 16 | settings = toml.load("settings.toml") 17 | 18 | 19 | 20 | # check 3 invariants: 21 | # 1. totalsupply 22 | # 2. totalBorrow 23 | # 3. totalReserve 24 | 25 | def removePaddingZeros(value: str): 26 | if value == "0x0000000000000000000000000000000000000000000000000000000000000000": 27 | return "0x0" 28 | else: 29 | # remove padding zeros 30 | temp = value 31 | if temp.startswith("0x"): 32 | temp = temp[2:] 33 | temp = temp.lstrip("0") 34 | temp = "0x" + temp 35 | return temp 36 | 37 | 38 | 39 | 40 | def contractSlot2Type(contract, slot): 41 | tag = None # totalSupply, totalBorrow 42 | 43 | if len(slot) > 7: 44 | return None 45 | 46 | # CheeseBank_1 47 | if contract == "0x5e181bdde2fa8af7265cb3124735e9a13779c021".lower(): 48 | # https://evm.storage/eth/18021920/0x5e181bdde2fa8af7265cb3124735e9a13779c021 49 | if slot == "0xb": # totalBorrow 50 | tag = "totalBorrow" 51 | elif slot == "0xc": # totalReserve 52 | tag = "totalReserve" 53 | elif slot == "0xd": # totalSupply 54 | tag = "totalSupply" 55 | 56 | # CheeseBank_2 57 | if contract == "0x4c2a8A820940003cfE4a16294B239C8C55F29695".lower(): 58 | # https://evm.storage/eth/18021920/0x4c2a8A820940003cfE4a16294B239C8C55F29695 59 | if slot == "0xb": # totalBorrow 60 | tag = "totalBorrow" 61 | elif slot == "0xc": # totalReserve 62 | tag = "totalReserve" 63 | elif slot == "0xd": # totalSupply 64 | tag = "totalSupply" 65 | 66 | # CheeseBank_3 67 | if contract == "0xa80e737ded94e8d2483ec8d2e52892d9eb94cf1f".lower(): 68 | # https://evm.storage/eth/18021920/ 69 | if slot == "0xb": # totalBorrow 70 | tag = "totalBorrow" 71 | elif slot == "0xc": # totalReserve 72 | tag = "totalReserve" 73 | elif slot == "0xd": # totalSupply 74 | tag = "totalSupply" 75 | 76 | 77 | # bZx2 78 | if contract == "0x85ca13d8496b2d22d6518faeb524911e096dd7e0".lower(): 79 | # https://evm.storage/eth/17931184/0x85ca13d8496b2d22d6518faeb524911e096dd7e0 80 | if slot == "0x1b": # totalSupply 81 | tag = "totalSupply" 82 | elif slot == "0x15": # totalAssetBorrow 83 | tag = "totalBorrow" 84 | elif slot == "0xc": # rateMultiplier 85 | tag = "variable" 86 | elif slot == "0xb": # baseRate 87 | tag = "variable" 88 | elif slot == "0x8": # loanTokenAddress 89 | tag = "variable" 90 | 91 | # Warp 92 | if contract == "0x6046c3Ab74e6cE761d218B9117d5c63200f4b406".lower(): 93 | # https://evm.storage/eth/17931184/0x6046c3Ab74e6cE761d218B9117d5c63200f4b406 94 | if slot == "0x4": # totalBorrow 95 | tag = "totalBorrow" 96 | elif slot == "0x5": # totalReserve 97 | tag = "totalReserve" 98 | 99 | # mainCreamFi1_1 100 | if contract == "0x3c710b981f5ef28da1807ce7ed3f2a28580e0754".lower(): 101 | # https://evm.storage/eth/17931184/0x3c710b981f5ef28da1807ce7ed3f2a28580e0754 102 | if slot == "0xa": # borrowIndex 103 | tag = "variable" 104 | elif slot == "0xd": # totalSupply 105 | tag = "totalSupply" 106 | elif slot == "0xb": # totalBorrows 107 | tag = "totalBorrow" 108 | elif slot == "0xc": # totalReserves 109 | tag = "totalReserve" 110 | elif slot == "0x13": # internalCash 111 | tag = "variable" 112 | 113 | 114 | # mainCreamFi1_2 115 | if contract == "0xd06527d5e56a3495252a528c4987003b712860ee".lower(): 116 | # https://evm.storage/eth/17931184/0xd06527d5e56a3495252a528c4987003b712860ee 117 | if slot == "0xb": # totalBorrows 118 | tag = "totalBorrow" 119 | elif slot == "0xc": # totalReserves 120 | tag = "totalReserve" 121 | elif slot == "0xd": # totalSupply 122 | tag = "totalSupply" 123 | 124 | # RariCapital1 125 | if contract == "0xec260f5a7a729bb3d0c42d292de159b4cb1844a3".lower(): 126 | # https://evm.storage/eth/17931184/0xec260f5a7a729bb3d0c42d292de159b4cb1844a3 127 | if slot == "0x70": # _netDeposits 128 | tag = "totalReserve" 129 | 130 | 131 | # RariCapital2_1, RariCapital2_2, RariCapital2_3, RariCapital2_4 132 | if contract == "0x67db14e73c2dce786b5bbbfa4d010deab4bbfcf9".lower(): 133 | # https://evm.storage/eth/17931184/0x67db14e73c2dce786b5bbbfa4d010deab4bbfcf9 134 | if slot == "0xc": # borrowIndex 135 | tag = "variable" 136 | # temporary fix 137 | elif slot == "0x5": # borrowRateMaxMantissa 138 | tag = "variable" 139 | elif slot == "0xb": # accrualBlockNumber 140 | tag = "variable" 141 | elif slot == "0xd": # totalBorrows 142 | tag = "totalBorrow" 143 | elif slot == "0xe": # totalReserves 144 | tag = "totalReserve" 145 | elif slot == "0xf": # totalAdminFees 146 | tag = "variable" 147 | elif slot == "0x10": # totalFuseFees 148 | tag = "variable" 149 | elif slot == "0x11": # totalSupply 150 | tag = "totalSupply" 151 | 152 | # mainXCarnival 153 | if contract == "0x5417da20ac8157dd5c07230cfc2b226fdcfc5663".lower(): 154 | # https://evm.storage/eth/17931184/0x5417da20ac8157dd5c07230cfc2b226fdcfc5663 155 | if slot == "0x3": # totalSupply 156 | tag = "totalSupply" 157 | elif slot == "0xc": # totalReserves 158 | tag = "totalReserve" 159 | elif slot == "0xb": # totalBorrows 160 | tag = "totalBorrow" 161 | elif slot == "0xf": # totalCash 162 | tag = "variable" 163 | elif slot == "0xd": # borrowIndex 164 | tag = "variable" 165 | 166 | # mainRariCapital2_1, mainRariCapital2_2, mainRariCapital2_3, mainRariCapital2_4 167 | if contract == "0xd77e28a1b9a9cfe1fc2eee70e391c05d25853cbf".lower(): 168 | # https://evm.storage/eth/17931184/0xd77e28a1b9a9cfe1fc2eee70e391c05d25853cbf 169 | if slot == "0x5": # borrowRateMaxMantissa 170 | tag = "variable" 171 | elif slot == "0xb": # accrualBlockNumber 172 | tag = "variable" 173 | elif slot == "0xd": # totalBorrows 174 | tag = "variable" 175 | tag = "totalBorrow" 176 | elif slot == "0xe": # totalReserves 177 | tag = "variable" 178 | tag = "totalReserve" 179 | elif slot == "0xf": # totalAdminFees 180 | tag = "variable" 181 | elif slot == "0x10": # totalFuseFees 182 | tag = "variable" 183 | elif slot == "0x11": # totalSupply 184 | tag = "totalSupply" 185 | 186 | # mainHarvest2_fUSDC and mainHarvest1_fUSDT 187 | if contract == "0x9b3be0cc5dd26fd0254088d03d8206792715588b".lower(): 188 | # https://evm.storage/eth/17931184/0x9b3be0cc5dd26fd0254088d03d8206792715588b 189 | # https://vscode.blockscan.com/ethereum/0x9b3be0cc5dd26fd0254088d03d8206792715588b#line1032 190 | if slot == "0x35": # _totalSupply 191 | tag = "totalSupply" 192 | 193 | # mainValueDeFi 194 | if contract == "0xddd7df28b1fb668b77860b473af819b03db61101".lower(): 195 | # https://evm.storage/eth/17931184/0xddd7df28b1fb668b77860b47sender3af819b03db61101 196 | if slot == "0x9e": # min 197 | tag = "variable" 198 | elif slot == "0x67": # _totalSupply 199 | tag = "totalSupply" 200 | elif slot == "0xa3": # insurance 201 | tag = "variable" 202 | elif slot == "0x97": # basedToken 203 | type = "address" 204 | 205 | # BeanstalkFarms 206 | if contract == "0x5f890841f657d90e081babdb532a05996af79fe6".lower(): 207 | # https://evm.storage/eth/17931184/0x5f890841f657d90e081babdb532a05996af79fe6 208 | if slot == "0x4": # fee 209 | tag = "variable" 210 | elif slot == "0xc": # rate_multiplier 211 | tag = "variable" 212 | elif slot == "0x9": # future_A 213 | tag = "variable" 214 | elif slot == "0x11": # totalSupply 215 | tag = "totalSupply" 216 | 217 | # Dodo 218 | if contract == "0x2bbd66fc4898242bdbd2583bbe1d76e8b8f71445".lower(): 219 | # https://evm.storage/eth/17931184/0x2bbd66fc4898242bdbd2583bbe1d76e8b8f71445 220 | if slot == "0x8": # totalSupply 221 | tag = "totalSupply" 222 | elif slot == "0x3": # _block_timestamp_last _QUOTE_RESERVE_ _BASE_RESERVE_ 223 | type = "uint" 224 | elif slot == "0x1": # _BASE_TOKEN_ 225 | type = "address" 226 | elif slot == "0x10": # _I_ 227 | tag = "variable" 228 | elif slot == "0xd": # _LP_FEE_RATE_ 229 | tag = "variable" 230 | elif slot == "0xf": # _K_ 231 | tag = "variable" 232 | # Yearn 233 | if contract == "0x9c211bfa6dc329c5e757a223fb72f5481d676dc1".lower(): 234 | # https://evm.storage/eth/17931184/0x9c211bfa6dc329c5e757a223fb72f5481d676dc1 235 | if slot == "0x7": # slip 236 | tag = "variable" 237 | elif slot == "0x8": # tank 238 | tag = "variable" 239 | elif slot == "0x4": # withdrawalFee 240 | tag = "variable" 241 | 242 | # Eminence: 243 | if contract == "0x5ade7aE8660293F2ebfcEfaba91d141d72d221e8".lower(): 244 | # https://evm.storage/eth/17931184/0x5ade7aE8660293F2ebfcEfaba91d141d72d221e8 245 | if slot == "0x2": # _totalSupply 246 | tag = "totalSupply" 247 | elif slot == "0x4": # reserveBalance 248 | tag = "totalReserve" 249 | elif slot == "0x5": # reserveRatio 250 | type = "uint32" 251 | 252 | # indexFi 253 | if contract == "0x5bd628141c62a901e0a83e630ce5fafa95bbdee4".lower(): 254 | # https://evm.storage/eth/17931184/0x5bd628141c62a901e0a83e630ce5fafa95bbdee4 255 | if slot == "0x7": # _swapFee 256 | tag = "variable" 257 | elif slot == "0x2": # _totalSupply 258 | tag = "totalSupply" 259 | elif slot == "0xa": # _totalWeight 260 | tag = "variable" 261 | 262 | # InverseFi 263 | if contract == "0x7Fcb7DAC61eE35b3D4a51117A7c58D53f0a8a670".lower(): 264 | # https://evm.storage/eth/17931184/0x7Fcb7DAC61eE35b3D4a51117A7c58D53f0a8a670 265 | if slot == "0xd": # totalSupply 266 | tag = "totalSupply" 267 | elif slot == "0xc": # totalReserves 268 | tag = "totalReserve" 269 | elif slot == "0xb": # totalBorrows 270 | tag = "totalBorrow" 271 | 272 | # # Opyn 273 | # if contract == "0x951d51baefb72319d9fbe941e1615938d89abfe2".lower(): 274 | # # https://evm.storage/eth/17931184/0x951d51baefb72319d9fbe941e1615938d89abfe2 275 | # if slot == "0x16": # 276 | # # IERC20 collateral 277 | # # int32 underlyingExp 278 | # # int32 collateralExp 279 | # tag = "variable" 280 | # elif slot == "0x10": # exponent 281 | # tag = "variable" 282 | # elif slot == "0xf": # value 283 | # tag = "variable" 284 | 285 | return tag 286 | 287 | 288 | 289 | 290 | def inferSpecialStorage(accesslistTable): 291 | crawlEtherscan = CrawlEtherscan() 292 | for contract, accessList in accesslistTable: 293 | if contract in proxyMap: 294 | contract = proxyMap[contract] 295 | # build read-only functions 296 | ABI = crawlEtherscan.Contract2ABI(contract) 297 | readOnlyFunctions = ["fallback"] 298 | nonReadOnlyFunctions = [] 299 | for function in ABI: 300 | if function["type"] == "function" and (function["stateMutability"] == "view" or function["stateMutability"] == "pure"): 301 | readOnlyFunctions.append(function["name"]) 302 | if function["type"] == "function" and (function["stateMutability"] != "view" and function["stateMutability"] != "pure"): 303 | nonReadOnlyFunctions.append(function["name"]) 304 | 305 | 306 | # stage 1: training 307 | specialStorageMap = {} 308 | # { 309 | # "slot": (type, [ value1, value2, ... ]) 310 | # } 311 | 312 | for tx, funcCallList in accessList: 313 | if len(funcCallList) != 1: 314 | print(funcCallList) 315 | sys.exit("money flow infer2: not one function call in a transaction") 316 | 317 | for funcCall in funcCallList[0]: 318 | contract = funcCall["addr"] 319 | proxy = funcCall["proxy"] if "proxy" in funcCall else None 320 | if "sstore" not in funcCall: 321 | continue 322 | for slot, value, pc in funcCall["sstore"]: 323 | if isinstance(value, str): 324 | value = int(value, 16) 325 | tag1 = contractSlot2Type(contract, slot) 326 | tag2 = contractSlot2Type(proxy, slot) 327 | tag = None 328 | if tag1 == "totalSupply" or tag2 == "totalSupply": 329 | tag = "totalSupply" 330 | elif tag1 == "totalBorrow" or tag2 == "totalBorrow": 331 | tag = "totalBorrow" 332 | # elif tag1 == "totalReserve" or tag2 == "totalReserve": 333 | # tag = "totalReserve" 334 | 335 | if tag is not None: 336 | if slot not in specialStorageMap: 337 | specialStorageMap[slot] = (tag, [value]) 338 | else: 339 | specialStorageMap[slot][1].append(value) 340 | 341 | 342 | 343 | 344 | invariantMap = {} 345 | # { 346 | # "totalSupply": (lowerBound, upperBound), 347 | # "totalBorrow": (lowerBound, upperBound), 348 | # } 349 | 350 | # stage 2: infer 351 | for slot in specialStorageMap: 352 | tag, values = specialStorageMap[slot] 353 | if tag not in invariantMap: 354 | invariantMap[tag] = (min(values), max(values)) 355 | else: 356 | sys.exit("special storage: multiple slots for the same tag") 357 | 358 | # print("==specialStorage map: ") 359 | # print(specialStorageMap) 360 | print("==invariant map: ") 361 | print(invariantMap) 362 | 363 | -------------------------------------------------------------------------------- /constraintPackage/timeLockInfer.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | from parserPackage.locator import * 4 | from trackerPackage.dataSource import * 5 | from fetchPackage.fetchTrace import fetcher 6 | from crawlPackage.crawlQuicknode import CrawlQuickNode 7 | from crawlPackage.crawlEtherscan import CrawlEtherscan 8 | from utilsPackage.compressor import * 9 | from staticAnalyzer.analyzer import Analyzer 10 | 11 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 12 | sys.path.append(os.path.dirname(SCRIPT_DIR)) 13 | 14 | import toml 15 | settings = toml.load("settings.toml") 16 | 17 | 18 | 19 | 20 | 21 | def inferTimeLocks(accesslistTable, enterFuncs, exitFuncs): 22 | 23 | crawlQuickNode = CrawlQuickNode() 24 | crawlEtherscan = CrawlEtherscan() 25 | 26 | 27 | for contract, accessList in accesslistTable: 28 | if any(i in enterFuncs for i in exitFuncs): 29 | sys.exit("timeLockInfer: enterFuncs and exitFuncs overlap") 30 | 31 | # build read-only functions 32 | ABI = crawlEtherscan.Contract2ABI(contract) 33 | readOnlyFunctions = ["fallback"] 34 | for function in ABI: 35 | if function["type"] == "function" and function["stateMutability"] == "view": 36 | readOnlyFunctions.append(function["name"]) 37 | 38 | # check 2 invariants: 39 | # 1. the same origin/sender cannot enters and exits the protocol in the same block 40 | # _lastCallerBlock = keccak256(abi.encodePacked(tx.origin, block.number)); 41 | # require(keccak256(abi.encodePacked(tx.origin, block.number)) != _lastCallerBlock, "8"); 42 | 43 | # 2. the same function cannot be called within a gap of N blocks 44 | # block - lastUpdate > constant 45 | 46 | # stage 1: training 47 | timeLocksMap = {} 48 | # { 49 | # "func": [block1, block2, ...], 50 | # } 51 | senderBlockPair = (0, 0) 52 | # (sender, block) 53 | originBlockPair = (0, 0) 54 | # (origin, block) 55 | 56 | invariantMap = { 57 | "checkSameSenderBlock": len(enterFuncs) > 0 and len(exitFuncs) > 0, 58 | "checkSameOriginBlock": len(enterFuncs) > 0 and len(exitFuncs) > 0, 59 | } 60 | # { 61 | # "func": (shortest_block_gap, lastCallBlock) 62 | # "checkSameSenderBlock": True/False 63 | # "checkSameOriginBlock": True/False 64 | # } 65 | 66 | 67 | counter = -1 68 | 69 | analyzer = Analyzer() 70 | for tx, funcCallList in accessList: 71 | counter += 1 72 | 73 | origin = crawlQuickNode.Tx2Details(tx)["from"].lower() 74 | block = crawlQuickNode.Tx2Block(tx) 75 | 76 | if len(funcCallList) != 1: 77 | print(funcCallList) 78 | sys.exit("access control infer: not one function call in a transaction") 79 | 80 | for funcCall in funcCallList[0]: 81 | sender = funcCall["msg.sender"].lower() 82 | name = "" 83 | if "name" in funcCall: 84 | if funcCall["name"] == "fallback" and funcCall["Selector"] != "0x": 85 | # get real funcName from selector and contract 86 | funcSigMap = analyzer.contract2funcSigMap(contract) 87 | funcCall["name"] = funcSigMap[funcCall["Selector"].lower()][0] 88 | name += funcCall["name"] 89 | if funcCall["name"] in readOnlyFunctions: 90 | continue 91 | # if funcCall["name"] not in enterFuncs + exitFuncs: 92 | # continue 93 | # if "Selector" in funcCall: 94 | # name += funcCall["Selector"] 95 | if name not in timeLocksMap: 96 | timeLocksMap[name] = [block] 97 | else: 98 | timeLocksMap[name].append(block) 99 | 100 | 101 | if name in enterFuncs: 102 | originBlockPair = (origin, block) 103 | senderBlockPair = (sender, block) 104 | if name in exitFuncs: 105 | if origin == originBlockPair[0] and block == originBlockPair[1]: 106 | invariantMap["checkSameOriginBlock"] = False 107 | if sender == senderBlockPair[0] and block == originBlockPair[1]: 108 | invariantMap["checkSameSenderBlock"] = False 109 | 110 | 111 | 112 | # stage 2: infer 113 | # invariant 1: the same origin/sender cannot enters and exits the protocol in the same block 114 | pass 115 | # i) checkSameSenderBlock 116 | # ii) checkSameOriginBlock 117 | 118 | # invariantMap = { 119 | # "func": (shortest_block_gap, lastCallBlock) 120 | # "checkSameSenderBlock": True/False 121 | # "checkSameOriginBlock": True/False 122 | # } 123 | 124 | 125 | # invariant 2: the same function cannot be called within a gap of N blocks 126 | for func in timeLocksMap: 127 | if len(timeLocksMap[func]) >= 2: 128 | shortestGap = 99999999999 129 | for i in range(len(timeLocksMap[func])-1): 130 | thisCall = timeLocksMap[func][i] 131 | nextCall = timeLocksMap[func][i+1] 132 | gap = nextCall - thisCall 133 | if gap < shortestGap: 134 | shortestGap = gap 135 | # print("shortestGap: {} for func {}".format(shortestGap, func), end = "") 136 | if shortestGap != 0: 137 | if func in invariantMap: 138 | sys.exit("invariantMap[func] already exists") 139 | else: 140 | invariantMap[func] = (shortestGap, timeLocksMap[func][-1]) 141 | 142 | 143 | print("==invariant map: ") 144 | import pprint 145 | pp = pprint.PrettyPrinter(indent=2) 146 | # print(timeLocksMap) 147 | pp.pprint(invariantMap) 148 | 149 | print("Interpretation of the above invariant map: ") 150 | for func in invariantMap: 151 | if func == "checkSameOriginBlock" or func == "checkSameSenderBlock": 152 | print("is the invariant {} satisfied? {}".format(func, invariantMap[func])) 153 | else: 154 | print("For the function {}, the shortest block gap is {} and the last call block is {}".format(func, invariantMap[func][0], invariantMap[func][1])) 155 | -------------------------------------------------------------------------------- /crawlPackage/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeffchen006/OpenTracer/6d3162c351051b82b1184aaaeb84034b0b1f959e/crawlPackage/.DS_Store -------------------------------------------------------------------------------- /crawlPackage/README.md: -------------------------------------------------------------------------------- 1 | This module is designed to crawl history transaction involving a contract address from Etherscan. 2 | 3 | input: a contract address 4 | output: 5 | (1) deployment block 6 | (2) a list of (block, transaction hash) in chronological order. 7 | 8 | An example -- Eminence: 9 | Victim contract: 0x5ade7aE8660293F2ebfcEfaba91d141d72d221e8 10 | Deploy block: 10950651 11 | Hack happen: 0x3503253131644dd9f52802d071de74e456570374d586ddd640159cf6fb9b8ad8 12 | Hack block: 10954411 13 | 14 | -------------------------------------------------------------------------------- /crawlPackage/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeffchen006/OpenTracer/6d3162c351051b82b1184aaaeb84034b0b1f959e/crawlPackage/__init__.py -------------------------------------------------------------------------------- /crawlPackage/cache/cacheDeployTx.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeffchen006/OpenTracer/6d3162c351051b82b1184aaaeb84034b0b1f959e/crawlPackage/cache/cacheDeployTx.pickle -------------------------------------------------------------------------------- /crawlPackage/cacheDatabase.py: -------------------------------------------------------------------------------- 1 | import json 2 | import sys 3 | 4 | 5 | def _save_transaction_receipt(receipt, cur, txHash, conn): 6 | # receipt is assumed to be a dictionary 7 | # and its in proper format 8 | transactionhash = receipt['transactionHash'] 9 | blockNumber = receipt['blockNumber'] 10 | if not isinstance(blockNumber, int): 11 | blockNumber = int(blockNumber, 16) 12 | contractAddress = receipt['contractAddress'] 13 | cumulativeGasUsed = receipt['cumulativeGasUsed'] 14 | if not isinstance(cumulativeGasUsed, int): 15 | cumulativeGasUsed = int(cumulativeGasUsed, 16) 16 | effectiveGasPrice = receipt['effectiveGasPrice'] 17 | if not isinstance(effectiveGasPrice, int): 18 | effectiveGasPrice = int(effectiveGasPrice, 16) 19 | from_ = receipt['from'] 20 | gasUsed = receipt['gasUsed'] 21 | if not isinstance(gasUsed, int): 22 | gasUsed = int(gasUsed, 16) 23 | 24 | status = receipt['status'] 25 | if isinstance(status, int): 26 | status = hex(status) 27 | to = receipt['to'] 28 | transactionIndex = receipt['transactionIndex'] 29 | if not isinstance(transactionIndex, int): 30 | transactionIndex = int(transactionIndex, 16) 31 | type = receipt['type'] 32 | input = receipt['input'] if 'input' in receipt else None 33 | value = receipt['value'] if 'value' in receipt else None 34 | 35 | 36 | data = ( 37 | transactionhash, 38 | blockNumber, 39 | contractAddress, 40 | cumulativeGasUsed, 41 | effectiveGasPrice, 42 | from_, 43 | gasUsed, 44 | status, 45 | to, 46 | transactionIndex, 47 | type, 48 | input, 49 | value 50 | ) 51 | # Insert the data into the transactions table 52 | cur.execute(''' 53 | INSERT OR REPLACE INTO transactions ( 54 | transactionHash, blockNumber, contractAddress, cumulativeGasUsed, 55 | effectiveGasPrice, fromAddress, gasUsed, status, toAddress, transactionIndex, type, input, value 56 | ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) 57 | ''', data) 58 | conn.commit() 59 | 60 | 61 | def _load_transaction_receipt(transaction_hash, cur): 62 | # Query the database for the transaction with the specified hash 63 | cur.execute(''' 64 | SELECT * FROM transactions WHERE transactionHash = ? 65 | ''', (transaction_hash,)) 66 | 67 | # Fetch the result 68 | transaction = cur.fetchone() 69 | 70 | if transaction: 71 | # Convert the transaction tuple to a more readable format (optional) 72 | transaction_dict = { 73 | 'transactionHash': transaction[0], 74 | 'blockNumber': transaction[1], 75 | 'transactionIndex': transaction[2], 76 | 'contractAddress': transaction[3], 77 | 'from': transaction[4], 78 | 'to': transaction[5], 79 | 'status': transaction[6], 80 | 'type': transaction[7], 81 | 'gasUsed': transaction[8], 82 | 'cumulativeGasUsed': transaction[9], 83 | 'effectiveGasPrice': transaction[10], 84 | 'input': transaction[11], 85 | 'value': transaction[12] 86 | } 87 | return transaction_dict 88 | else: 89 | return None 90 | 91 | 92 | def _save_contract(receipt, cur, contractAddress, conn): 93 | # {'SourceCode': '', 'ABI': 'Contract source code not verified', 'ContractName': '', 'CompilerVersion': '', 'OptimizationUsed': '', 'Runs': '', 'ConstructorArguments': '', 'EVMVersion': 'Default', 'Library': '', 'LicenseType': 'Unknown', 'Proxy': '0', 'Implementation': '', 'SwarmSource': ''} 94 | # receipt is assumed to be a dictionary 95 | if not isinstance(receipt, dict): 96 | sys.exit("Error: receipt is not a dictionary") 97 | if 'SourceCode' not in receipt or 'ABI' not in receipt or \ 98 | 'ContractName' not in receipt or 'CompilerVersion' not in receipt or \ 99 | 'OptimizationUsed' not in receipt or 'Runs' not in receipt or \ 100 | 'ConstructorArguments' not in receipt or 'EVMVersion' not in receipt or \ 101 | 'Library' not in receipt or 'LicenseType' not in receipt or \ 102 | 'Proxy' not in receipt or 'Implementation' not in receipt or \ 103 | 'SwarmSource' not in receipt: 104 | sys.exit("Error: receipt is missing a key") 105 | 106 | SourceCode = receipt['SourceCode'] 107 | ABI = receipt['ABI'] 108 | ContractName = receipt['ContractName'] 109 | CompilerVersion = receipt['CompilerVersion'] 110 | OptimizationUsed = receipt['OptimizationUsed'] 111 | Runs = receipt['Runs'] 112 | ConstructorArguments = receipt['ConstructorArguments'] 113 | EVMVersion = receipt['EVMVersion'] 114 | Library = receipt['Library'] 115 | LicenseType = receipt['LicenseType'] 116 | Proxy = receipt['Proxy'] 117 | Implementation = receipt['Implementation'] 118 | SwarmSource = receipt['SwarmSource'] 119 | 120 | data = ( 121 | contractAddress, 122 | SourceCode, 123 | ABI, 124 | ContractName, 125 | CompilerVersion, 126 | OptimizationUsed, 127 | Runs, 128 | ConstructorArguments, 129 | EVMVersion, 130 | Library, 131 | LicenseType, 132 | Proxy, 133 | Implementation, 134 | SwarmSource 135 | ) 136 | 137 | 138 | # Insert the data into the contracts table 139 | cur.execute(''' 140 | INSERT OR REPLACE INTO contracts ( 141 | contractAddress, SourceCode, ABI, ContractName, CompilerVersion, OptimizationUsed, Runs, 142 | ConstructorArguments, EVMVersion, Library, LicenseType, Proxy, Implementation, SwarmSource 143 | ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) 144 | ''', data) 145 | conn.commit() 146 | pass 147 | 148 | def _load_contract(contractAddress, cur): 149 | # Query the database for the contract with the specified address 150 | cur.execute(''' 151 | SELECT * FROM contracts WHERE contractAddress = ? 152 | ''', (contractAddress,)) 153 | 154 | # Fetch the result 155 | contract = cur.fetchone() 156 | if contract: 157 | # Convert the contract tuple to a more readable format (optional) 158 | contract_dict = { 159 | 'contractAddress': contract[0], 160 | 'SourceCode': contract[1], 161 | 'ABI': contract[2], 162 | 'ContractName': contract[3], 163 | 'CompilerVersion': contract[4], 164 | 'OptimizationUsed': contract[5], 165 | 'Runs': contract[6], 166 | 'ConstructorArguments': contract[7], 167 | 'EVMVersion': contract[8], 168 | 'Library': contract[9], 169 | 'LicenseType': contract[10], 170 | 'Proxy': contract[11], 171 | 'Implementation': contract[12], 172 | 'SwarmSource': contract[13] 173 | } 174 | return contract_dict 175 | else: 176 | return None 177 | -------------------------------------------------------------------------------- /crawlPackage/crawl.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from crawlPackage.crawlEtherscan import CrawlEtherscan 4 | from crawlPackage.crawlQuicknode import CrawlQuickNode 5 | from crawlPackage.crawlTrueBlocks import CrawlTrueBlocks 6 | import subprocess 7 | import json 8 | import time 9 | import random 10 | import multiprocessing 11 | from multiprocessing import Process, Manager 12 | import copy 13 | 14 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 15 | sys.path.append(os.path.dirname(SCRIPT_DIR)) 16 | 17 | 18 | class Crawler: 19 | def __init__(self) -> None: 20 | self.crawlEtherscan = CrawlEtherscan() 21 | self.crawlQuicknode = CrawlQuickNode() 22 | self.crawlTrueBlocks = CrawlTrueBlocks() 23 | 24 | def Contract2TxHistoryBlockIndex(self, contractAddress: str, endBlock: int = 999999999999999, maxTxs: int = None) -> list: 25 | """Given a contract address and an end block, return a list of (block, blockIndex) tuples""" 26 | ContractTxHistoryBlockIndex = CrawlTrueBlocks().Contract2TxHistoryBlockIndex(contractAddress) 27 | # find number of elements before endBlock 28 | num = 0 29 | for i in range(len(ContractTxHistoryBlockIndex)): 30 | num += 1 31 | if ContractTxHistoryBlockIndex[i][0] > endBlock: 32 | break 33 | index = 0 34 | for ii in range(len(ContractTxHistoryBlockIndex)): 35 | block, _ = ContractTxHistoryBlockIndex[ii] 36 | if block <= endBlock: 37 | index = ii 38 | end = index + 1 39 | if maxTxs != None: 40 | end = min(index + 1, maxTxs) 41 | return ContractTxHistoryBlockIndex[0: end] 42 | 43 | 44 | def BlockIndex2TxHelper(self, aa, bb, index, managerList): 45 | Tx = self.BlockIndex2Tx(aa, bb) 46 | managerList[index] = Tx 47 | 48 | def Contract2TxHistory(self, contractAddress: str, endBlock: int = 999999999999999, maxTxs: int = None) -> list: 49 | """Given a contract address and an end block, return a list of tx hashes from deploy block to end block""" 50 | HistoryBlockIndex = self.Contract2TxHistoryBlockIndex(contractAddress, endBlock, maxTxs) 51 | # remove duplicates in HistoryBlockIndex 52 | new_HistoryBlockIndex = [] 53 | for HistoryBlock in HistoryBlockIndex: 54 | if HistoryBlock not in new_HistoryBlockIndex: 55 | new_HistoryBlockIndex.append(HistoryBlock) 56 | HistoryBlockIndex = new_HistoryBlockIndex 57 | 58 | txHashes = [] 59 | index = 0 60 | batchSize = 1000 61 | timegap = 1 62 | while True: 63 | start = index * batchSize 64 | end = min( (index + 1) * batchSize, len(HistoryBlockIndex) ) 65 | Txs = None 66 | while Txs == None: 67 | try: 68 | Txs = self.crawlQuicknode.batch_BlockIndex2Tx(HistoryBlockIndex[start: end]) 69 | except Exception as e: 70 | pass 71 | txHashes += Txs 72 | if end == len(HistoryBlockIndex): 73 | break 74 | index += 1 75 | print("start: " + str(start) + " end: " + str(end)) 76 | txHashes = self.blockIndexes2Txs(HistoryBlockIndex) 77 | return txHashes 78 | 79 | 80 | 81 | def blockIndexes2Txs(self, HistoryBlockIndex): 82 | # if len(HistoryBlockIndex) > 100000: 83 | # print("but we only crawl the last 100000 txs") 84 | # HistoryBlockIndex = copy.deepcopy(HistoryBlockIndex[-100000:]) 85 | 86 | # if len(HistoryBlockIndex) > 100000: 87 | # print("temporally skip crawling tx history") 88 | # return 89 | 90 | 91 | txHashes = [] 92 | index = 0 93 | batchSize = 1000 94 | timegap = 1 95 | while True: 96 | start = index * batchSize 97 | end = min( (index + 1) * batchSize, len(HistoryBlockIndex) ) 98 | Txs = None 99 | while Txs == None: 100 | try: 101 | Txs = self.crawlQuicknode.batch_BlockIndex2Tx(HistoryBlockIndex[start: end]) 102 | # print("HistoryBlockIndex", HistoryBlockIndex[start: end]) 103 | # print("Txs", Txs) 104 | except Exception as e: 105 | # time.sleep(timegap) 106 | # timegap += 2 107 | pass 108 | txHashes += Txs 109 | if end == len(HistoryBlockIndex): 110 | break 111 | index += 1 112 | print("start: " + str(start) + " end: " + str(end)) 113 | return txHashes 114 | 115 | 116 | 117 | 118 | def Contract2TxHistoryRankedbyGasUsed(self, contractAddress: str, endBlock: int = 999999999999999) -> list: 119 | """Given a contract address and an end block, return a list of tx hashes from deploy block to end block, ranked by gas used""" 120 | ContractTxHistoryBlockIndex = self.crawlTrueBlocks.Contract2TxHistoryBlockIndex(contractAddress) 121 | txHashes = [] 122 | # find number of elements before endBlock 123 | num = 0 124 | for i in range(len(ContractTxHistoryBlockIndex)): 125 | num += 1 126 | if ContractTxHistoryBlockIndex[i][0] > endBlock: 127 | break 128 | 129 | for ii in range(len(ContractTxHistoryBlockIndex)): 130 | (block, blockIndex) = ContractTxHistoryBlockIndex[ii] 131 | if block <= endBlock: 132 | txHash, gasUsed = self.BlockIndex2TxGasUsed(block, blockIndex) 133 | txHashes.append((txHash, gasUsed)) 134 | if ii % 100 == 0: 135 | print("working on {} out of {} txs".format(ii, num)) 136 | else: 137 | break 138 | 139 | txHashes.sort(key=lambda x: x[1], reverse=True) 140 | 141 | return txHashes 142 | 143 | def Tx2Block(self, Tx: str) -> int: 144 | """Given a tx hash, return its block number""" 145 | return self.crawlQuicknode.Tx2Block(Tx) 146 | 147 | def Tx2BlockIndex(self, Tx: str) -> int: 148 | """Given a tx hash, return its block index""" 149 | return self.crawlQuicknode.Tx2BlockIndex(Tx) 150 | 151 | def BlockIndex2Tx(self, block: int, blockIndex: int) -> str: 152 | """Given a block number and a block index, return the tx hash""" 153 | return self.crawlQuicknode.BlockIndex2Tx(block, blockIndex) 154 | 155 | def BlockIndex2TxGasUsed(self, block: int, blockIndex: int) -> int: 156 | """Given a block number and a block index, return the tx gas used""" 157 | return self.crawlQuicknode.BlockIndex2TxGasUsed(block, blockIndex) 158 | 159 | 160 | 161 | 162 | 163 | def main(): 164 | crawler = Crawler() 165 | # test Contract2TxHistory 166 | contractAddress = '0xcc13fc627effd6e35d2d2706ea3c4d7396c610ea' 167 | 168 | contractAddress = "0x11111254369792b2Ca5d084aB5eEA397cA8fa48B" # 1inch exchange 2 169 | 170 | start_time = time.time() 171 | 172 | txHashes = crawler.Contract2TxHistory(contractAddress) 173 | print(txHashes) 174 | end_time = time.time() - start_time 175 | print("In total, it takes {} seconds".format(end_time)) 176 | 177 | 178 | # test Contract2TxHistoryRankedbyGasUsed 179 | # start_time = time.time() 180 | # txHistory = crawler.Contract2TxHistoryRankedbyGasUsed(contractAddress, 4925047 + 10000) 181 | # end_time = time.time() - start_time 182 | # print("In total, it takes {} seconds to get Contract2TxHistoryRankedbyGasUsed".format(end_time)) 183 | 184 | # print("The first 10 txs are:") 185 | # for i in range(10): 186 | # print(txHistory[i]) 187 | 188 | # start_time = time.time() 189 | # txHistoryRankedByGasUsed = crawler.TxHistory2RankedByGasUsed(txHistory) 190 | # end_time = time.time() - start_time 191 | # print("In total, it takes {} seconds to get txHistoryRankedByGasUsed".format(end_time)) 192 | 193 | 194 | 195 | 196 | if __name__ == '__main__': 197 | main() 198 | -------------------------------------------------------------------------------- /crawlPackage/crawlTrueBlocks.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import json 3 | import sys, os 4 | from utilsPackage.compressor import * 5 | 6 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 7 | sys.path.append(os.path.dirname(SCRIPT_DIR)) 8 | 9 | class CrawlTrueBlocks: 10 | def __init__(self): 11 | pass 12 | 13 | def Block2Txs(self, block: int) -> list: 14 | """Given a block number, return a list of tx hashes in that block""" 15 | output = subprocess.getoutput("chifra blocks -e " + str(block)) 16 | output_json = json.loads(output) 17 | txHashes = output_json['data'][0]['tx_hashes'] 18 | return txHashes 19 | 20 | def BlockIndex2TxHash(self, block: int, blockIndex: int) -> str: 21 | """Given a block number and a block index, return the tx hash""" 22 | txHashes = self.Block2TxHashes(block) 23 | return txHashes[blockIndex] 24 | 25 | def Contract2TxHistoryBlockIndex(self, contractAddress: str) -> list: 26 | """Given a contract address, return a list of (block, blockIndex) tuples""" 27 | self.Contract2TxHistoryWarmUp(contractAddress) 28 | output = subprocess.getoutput("chifra list --fmt json " + contractAddress) 29 | output_json = json.loads(output) 30 | ContractTxHistoryBlockIndex = [] 31 | for data in output_json['data']: 32 | block = int(data['blockNumber']) 33 | blockIndex = int(data['transactionIndex']) 34 | ContractTxHistoryBlockIndex.append((block, blockIndex)) 35 | return ContractTxHistoryBlockIndex 36 | 37 | 38 | def Contract2TxHistoryWarmUp(self, contractAddress: str) -> None: 39 | """Given a contract address, warm up the cache""" 40 | subprocess.getoutput("chifra list --fmt json " + contractAddress) 41 | 42 | 43 | 44 | 45 | 46 | def main(): 47 | tb = CrawlTrueBlocks() 48 | # test Tx2GasUsed 49 | # txs = '0x04b713fdbbf14d4712df5ccc7bb3dfb102ac28b99872506a363c0dcc0ce4343c' 50 | # gasUsed = tb.Tx2GasUsed(txs) 51 | # print(gasUsed) 52 | 53 | # test Contract2TxHistoryBlockIndex 54 | # contractAddress = '0xbEbc44782C7dB0a1A60Cb6fe97d0b483032FF1C7' 55 | # ContractTxHistoryBlockIndex = tb.Contract2TxHistoryBlockIndex(contractAddress) 56 | 57 | # blockBlockIndexList = [] 58 | 59 | # for block, blockIndex in ContractTxHistoryBlockIndex: 60 | # if block >= 11687459 and block <= 11792184: 61 | # blockBlockIndexList.append((block, blockIndex)) 62 | # writeCompressedJson('3PoolBlockIndexList.pickle', blockBlockIndexList) 63 | # blockBlockIndexList = readCompressedJson('3PoolBlockIndexList.pickle') 64 | # print(len(blockBlockIndexList)) 65 | 66 | 67 | contract = "0x72AFAECF99C9d9C8215fF44C77B94B99C28741e8" # Chainlink Oracle 68 | LIST = tb.Contract2TxHistoryBlockIndex(contract) 69 | print(LIST) 70 | 71 | 72 | 73 | if __name__ == '__main__': 74 | main() 75 | 76 | -------------------------------------------------------------------------------- /crawlPackage/database/etherScan.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeffchen006/OpenTracer/6d3162c351051b82b1184aaaeb84034b0b1f959e/crawlPackage/database/etherScan.db -------------------------------------------------------------------------------- /fetchPackage/StackCarpenter.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | class stackCarpener: 4 | 5 | def __init__(self) -> None: 6 | """opcodeInputStackmap: opcode -> length of input required by that opcode""" 7 | """opcodeOutputStackmap: opcode -> length of output emitted by that opcode""" 8 | 9 | self.opcodeInputStackmap = { 10 | "STOP": 0, "ADD": 2, 11 | "MUL": 2, "SUB": 2, 12 | "DIV": 2, "SDIV": 2, 13 | "MOD": 2, "SMOD": 2, 14 | "ADDMOD": 3, "MULMOD": 3, 15 | "EXP": 2, "SIGNEXTEND": 2, 16 | "LT": 2, "GT": 2, 17 | "SLT": 2, "SGT": 2, 18 | "EQ": 2, "ISZERO": 1, 19 | "AND": 2, "OR": 2, 20 | "XOR": 2, "NOT": 1, 21 | "BYTE": 2, "SHL": 2, 22 | "SHR": 2, "SAR": 2, 23 | "SHA3": 2, "KECCAK256": 2, 24 | "ADDRESS": 0, 25 | "BALANCE": 1, "ORIGIN": 0, 26 | "CALLER": 0, "CALLVALUE": 0, 27 | "CALLDATALOAD": 1, "CALLDATASIZE": 0, 28 | "CALLDATACOPY": 3, "CODESIZE": 0, 29 | "CODECOPY": 3, "GASPRICE": 0, 30 | "EXTCODESIZE": 1, "EXTCODECOPY": 4, 31 | "RETURNDATASIZE": 0, "RETURNDATACOPY": 3, 32 | "EXTCODEHASH": 1, "BLOCKHASH": 1, 33 | "COINBASE": 0, "TIMESTAMP": 0, 34 | "NUMBER": 0, "DIFFICULTY": 0, 35 | "GASLIMIT": 0, "CHAINID": 0, 36 | "SELFBALANCE": 0, "BASEFEE": 0, 37 | "POP": 1, "MLOAD": 1, 38 | "MSTORE": 2, "MSTORE8": 2, 39 | "SLOAD": 1, "SSTORE": 2, 40 | "JUMP": 1, "JUMPI": 2, 41 | "PC": 0, "MSIZE": 0, 42 | "GAS": 0, "JUMPDEST": 0, 43 | "PUSH0": 0, 44 | "PUSH1": 0, "PUSH2": 0, "PUSH3": 0, "PUSH4": 0, "PUSH5": 0, "PUSH6": 0, 45 | "PUSH7": 0, "PUSH8": 0, "PUSH9": 0, "PUSH10": 0, "PUSH11": 0, "PUSH12": 0, 46 | "PUSH13": 0, "PUSH14": 0, "PUSH15": 0, "PUSH16": 0, "PUSH17": 0, "PUSH18": 0, 47 | "PUSH19": 0, "PUSH20": 0, "PUSH21": 0, "PUSH22": 0, "PUSH23": 0, "PUSH24": 0, 48 | "PUSH25": 0, "PUSH26": 0, "PUSH27": 0, "PUSH28": 0, "PUSH29": 0, "PUSH30": 0, 49 | "PUSH31": 0, "PUSH32": 0, 50 | "DUP1": 1, "DUP2": 2, "DUP3": 3, "DUP4": 4, "DUP5": 5, "DUP6": 6, 51 | "DUP7": 7, "DUP8": 8, "DUP9": 9, "DUP10": 10, "DUP11": 11, "DUP12": 12, 52 | "DUP13": 13, "DUP14": 14, "DUP15": 15, "DUP16": 16, "SWAP1": 2, "SWAP2": 3, 53 | "SWAP3": 4, "SWAP4": 5, "SWAP5": 6, "SWAP6": 7, "SWAP7": 8, "SWAP8": 9, 54 | "SWAP9": 10, "SWAP10": 11, "SWAP11": 12, "SWAP12": 13, "SWAP13": 14, "SWAP14": 15, 55 | "SWAP15": 16, "SWAP16": 17, 56 | "LOG0": 2, "LOG1": 3, "LOG2": 4, "LOG3": 5, "LOG4": 6, 57 | "CREATE": 3, 58 | "CALL": 7, "CALLCODE": 7, 59 | "RETURN": 2, "DELEGATECALL": 6, 60 | "CREATE2": 4, "STATICCALL": 6, 61 | "REVERT": 2, "SELFDESTRUCT": 1, 62 | "INVALID": 0, "opcode 0xfe not defined": 0 63 | } 64 | 65 | self.opcodeOutputStackmap = { 66 | "STOP": 0, "ADD": 1, 67 | "MUL": 1, "SUB": 1, 68 | "DIV": 1, "SDIV": 1, 69 | "MOD": 1, "SMOD": 1, 70 | "ADDMOD": 1, "MULMOD": 1, 71 | "EXP": 1, "SIGNEXTEND": 1, 72 | "LT": 1, "GT": 1, 73 | "SLT": 1, "SGT": 1, 74 | "EQ": 1, "ISZERO": 1, 75 | "AND": 1, "OR": 1, 76 | "XOR": 1, "NOT": 1, 77 | "BYTE": 1, "SHL": 1, 78 | "SHR": 1, "SAR": 1, 79 | "SHA3": 1, "KECCAK256": 1, 80 | "ADDRESS": 1, 81 | "BALANCE": 1, "ORIGIN": 1, 82 | "CALLER": 1, "CALLVALUE": 1, 83 | "CALLDATALOAD": 1, "CALLDATASIZE": 1, 84 | "CALLDATACOPY": 0, "CODESIZE": 1, 85 | "CODECOPY": 0, "GASPRICE": 1, 86 | "EXTCODESIZE": 1, "EXTCODECOPY": 0, 87 | "RETURNDATASIZE": 1, "RETURNDATACOPY": 0, 88 | "EXTCODEHASH": 1, "BLOCKHASH": 1, 89 | "COINBASE": 1, "TIMESTAMP": 1, 90 | "NUMBER": 1, "DIFFICULTY": 1, 91 | "GASLIMIT": 1, "CHAINID": 1, 92 | "SELFBALANCE": 1, "BASEFEE": 1, 93 | "POP": 0, "MLOAD": 1, 94 | "MSTORE": 0, "MSTORE8": 0, 95 | "SLOAD": 1, "SSTORE": 0, 96 | "JUMP": 0, "JUMPI": 0, 97 | "PC": 1, "MSIZE": 1, 98 | "GAS": 1, "JUMPDEST": 0, 99 | "PUSH0": 1, 100 | "PUSH1": 1, "PUSH2": 1, "PUSH3": 1, "PUSH4": 1, "PUSH5": 1, "PUSH6": 1, 101 | "PUSH7": 1, "PUSH8": 1, "PUSH9": 1, "PUSH10": 1, "PUSH11": 1, "PUSH12": 1, 102 | "PUSH13": 1, "PUSH14": 1, "PUSH15": 1, "PUSH16": 1, "PUSH17": 1, "PUSH18": 1, 103 | "PUSH19": 1, "PUSH20": 1, "PUSH21": 1, "PUSH22": 1, "PUSH23": 1, "PUSH24": 1, 104 | "PUSH25": 1, "PUSH26": 1, "PUSH27": 1, "PUSH28": 1, "PUSH29": 1, "PUSH30": 1, 105 | "PUSH31": 1, "PUSH32": 1, 106 | "DUP1": 2, "DUP2": 3, "DUP3": 4, "DUP4": 5, "DUP5": 6, "DUP6": 7, 107 | "DUP7": 8, "DUP8": 9, "DUP9": 10, "DUP10": 11, "DUP11": 12, "DUP12": 13, 108 | "DUP13": 14, "DUP14": 15, "DUP15": 16, "DUP16": 17, 109 | "SWAP1": 2, "SWAP2": 3, "SWAP3": 4, "SWAP4": 5, "SWAP5": 6, "SWAP6": 7, 110 | "SWAP7": 8, "SWAP8": 9, "SWAP9": 10, "SWAP10": 11, "SWAP11": 12, "SWAP12": 13, 111 | "SWAP13": 14, "SWAP14": 15, "SWAP15": 16, "SWAP16": 17, 112 | "LOG0": 0, "LOG1": 0, "LOG2": 0, "LOG3": 0, "LOG4": 0, 113 | "CREATE": 1, "CALL": 1, "CALLCODE": 1, 114 | "RETURN": 0, "DELEGATECALL": 1, 115 | "CREATE2": 1, "STATICCALL": 1, 116 | "REVERT": 0, "SELFDESTRUCT": 0, 117 | "INVALID": 0, "opcode 0xfe not defined": 0 118 | } 119 | 120 | def opcode2InputStackLength(self, opcode: str) -> int: 121 | """Given an opcode, return the stack length needed to execute the opcode""" 122 | if opcode in self.opcodeInputStackmap: 123 | return self.opcodeInputStackmap[opcode] 124 | else: 125 | sys.exit("Opcode {} not found in opcode2InputStackLength!!".format(opcode)) 126 | 127 | def opcode2OutputStackLength(self, opcode: str) -> int: 128 | """Given an opcode, return the stack length after executing the opcode""" 129 | if opcode in self.opcodeOutputStackmap: 130 | return self.opcodeOutputStackmap[opcode] 131 | else: 132 | sys.exit("Opcode {} not found in opcode2OutputStackLength!!".format(opcode)) 133 | -------------------------------------------------------------------------------- /fetchPackage/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeffchen006/OpenTracer/6d3162c351051b82b1184aaaeb84034b0b1f959e/fetchPackage/__init__.py -------------------------------------------------------------------------------- /fetchPackage/fetchTrace.py: -------------------------------------------------------------------------------- 1 | from re import L 2 | from web3 import Web3, HTTPProvider 3 | import sys 4 | import os 5 | import toml 6 | settings = toml.load("settings.toml") 7 | import json 8 | from typing import Dict, List, Tuple 9 | import requests 10 | import multiprocessing 11 | import time 12 | import pickle 13 | import gzip 14 | import gc 15 | import random 16 | 17 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 18 | sys.path.append(os.path.dirname(SCRIPT_DIR)) 19 | from fetchPackage.StackCarpenter import stackCarpener 20 | from parserPackage.functions import * 21 | from utilsPackage.compressor import writeCompressedJson, readCompressedJson 22 | 23 | 24 | class fetcher: 25 | def __init__(self): 26 | self.urls = settings["settings"]["rpcProviders"] 27 | 28 | self.w3s = [] 29 | for url in self.urls: 30 | self.w3s.append(Web3(HTTPProvider(url, request_kwargs={'timeout': 60}))) 31 | self.counter = random.randint(0, len(self.urls)) 32 | self.stackCarpenter = stackCarpener() 33 | self.debug_traceTransactionSettings = { 34 | 'enableMemory': True, 35 | "disableMemory": False, 36 | 'disableStack': False, 37 | 'disableStorage': True, 38 | 'enableReturnData': False, 39 | } 40 | self.results = [] 41 | 42 | def get_url(self): 43 | self.counter += 1 44 | numOfUrls = len(self.urls) 45 | return self.urls[self.counter % numOfUrls] 46 | 47 | def get_w3(self): 48 | self.counter += 1 49 | numOfUrls = len(self.urls) 50 | return self.w3s[self.counter % numOfUrls] 51 | 52 | def pruneStructLog(self, structLog: dict, lastOpcode: str = None, FullTrace: bool = False): 53 | structLog_copy = structLog.copy() 54 | prune1 = True 55 | prune2 = True # still need it, otherwise exceed 100MB file size limit 56 | if FullTrace: 57 | prune1 = False 58 | prune2 = False 59 | 60 | # Prune 1: remove pc, and gasCost 61 | if prune1: 62 | # del structLog_copy["pc"] 63 | # del structLog_copy["gas"] 64 | # del structLog_copy["depth"] 65 | del structLog_copy["gasCost"] 66 | 67 | # Prune 2: remove unnessary stack (won't be used by opcode) 68 | if prune2: 69 | len1 = self.stackCarpenter.opcode2InputStackLength(structLog_copy["op"]) 70 | len2 = 0 71 | if lastOpcode != None: 72 | len2 = self.stackCarpenter.opcode2OutputStackLength(lastOpcode) 73 | necessaryStackLen = max(len1, len2) 74 | del structLog_copy["stack"][:-necessaryStackLen] 75 | for ii in range(len(structLog_copy["stack"])): 76 | if not structLog_copy["stack"][ii].startswith("0x"): 77 | # if structLog["stack"] contains a list of string of integers, convert them to string of hex integers 78 | structLog_copy["stack"][ii] = str(hex(int(structLog_copy["stack"][ii]))) 79 | 80 | 81 | if "error" in structLog_copy: 82 | error_dict = dict(structLog_copy["error"]).copy() 83 | structLog_copy["error"] = error_dict 84 | 85 | # Prune 3: remove unnessary memory (won't be used by opcode) 86 | if structLog_copy["op"] == "RETURN" \ 87 | or structLog_copy["op"] == "REVERT" \ 88 | or structLog_copy["op"] == "KECCAK256" \ 89 | or structLog_copy["op"] == "CODECOPY" \ 90 | or structLog_copy["op"] == "EXTCODECOPY" \ 91 | or structLog_copy["op"] == "RETURNDATACOPY" \ 92 | or structLog_copy["op"] == "SHA3" : 93 | pass 94 | elif structLog_copy["op"] == "CREATE" or structLog_copy["op"] == "CREATE2" or \ 95 | structLog_copy["op"] == "CALL" or structLog_copy["op"] == "CALLCODE" or \ 96 | structLog_copy["op"] == "STATICCALL" or structLog_copy["op"] == "DELEGATECALL": 97 | pass 98 | elif structLog_copy["op"] == "RETURN" or structLog_copy["op"] == "REVERT" or \ 99 | structLog_copy["op"] == "STOP" or structLog_copy["op"] == "SELFDESTRUCT" or \ 100 | structLog_copy["op"] == "INVALID": 101 | pass 102 | elif lastOpcode == "CALLDATACOPY" or lastOpcode == "CODECOPY" or lastOpcode == "EXTCODECOPY" or \ 103 | lastOpcode == "RETURNDATACOPY": 104 | pass 105 | elif lastOpcode == "CALL" or lastOpcode == "CALLCODE" or lastOpcode == "STATICCALL" or lastOpcode == "DELEGATECALL": 106 | pass 107 | 108 | else: 109 | if "memory" in structLog_copy: 110 | del structLog_copy["memory"] 111 | 112 | return structLog_copy 113 | 114 | def prettyPrintTrace(self, trace: dict): 115 | """Pretty print the trace data (returned by getTrace)""" 116 | print("structLogs:") 117 | for structLog in trace["structLogs"]: 118 | print(structLog) 119 | print("gas:", trace["gas"]) 120 | print("failed:", trace["failed"]) 121 | print("returnValue:", trace["returnValue"]) 122 | 123 | def batchRequests(self, calls: List[Tuple[str, List]]): 124 | payload = [ 125 | {"method": method, "params": params, "id": None, "jsonrpc": "2.0"} 126 | for method, params in calls 127 | ] 128 | batch_repsonse = requests.post(self.get_url(), json=payload).json() 129 | for response in batch_repsonse: 130 | if "error" in response: 131 | raise ValueError(response["error"]["message"]) 132 | yield response["result"] 133 | 134 | def cookStoreTrace(self, result, FullTrace: bool, category: str, contract: str, TxHash: str): 135 | result_dict = self.cookResult(result, FullTrace=FullTrace) 136 | path = getPathFromCategoryTxHash(category, contract, TxHash) 137 | writeCompressedJson(path, result_dict) 138 | 139 | def cookStoreTrace2(self, index: str, FullTrace: bool, category: str, contract: str, TxHash: str): 140 | result_dict = self.cookResult(self.results[index], FullTrace=FullTrace) 141 | path = getPathFromCategoryTxHash(category, contract, TxHash) 142 | writeCompressedJson(path, result_dict) 143 | 144 | def cookStoreTraces(self, indexes: str, FullTrace: bool, category: str, contract: str, TxHashes: str): 145 | for i in range(len(indexes)): 146 | result_dict = self.cookResult(self.results[indexes[i]], FullTrace) 147 | path = getPathFromCategoryTxHash(category, contract, TxHashes[i]) 148 | writeCompressedJson(path, result_dict) 149 | 150 | def batch_storeTrace3(self, category: str, contract: str, Txs: list, FullTrace: bool = False): 151 | """Given a list of tx hashes, store the trace data""" 152 | start = time.time() 153 | calls = [('debug_traceTransaction', [Tx, self.debug_traceTransactionSettings]) for Tx in Txs] 154 | results = self.batchRequests(calls) 155 | end = time.time() - start 156 | print("batch_getTrace time:", end) 157 | self.results = list(results) 158 | processNum = 25 159 | all_jobs = [] 160 | all_Txs = [] 161 | for i in range(processNum): 162 | all_jobs.append([]) 163 | all_Txs.append([]) 164 | for i in range(len(Txs)): 165 | all_jobs[i % processNum].append(i) 166 | all_Txs[i % processNum].append(Txs[i]) 167 | all_processes = [] 168 | for i in range(processNum): 169 | p = multiprocessing.Process(target=self.cookStoreTraces, args=(all_jobs[i], FullTrace, category, contract, all_Txs[i])) 170 | all_processes.append(p) 171 | p.start() 172 | for p in all_processes: 173 | p.join() 174 | all_processes = [] 175 | end2 = time.time() - (end + start) 176 | print("batch_storeTrace time:", end2) 177 | self.results = [] 178 | gc.collect() 179 | 180 | def batch_storeTrace2(self, category: str, contract: str, Txs: list, FullTrace: bool = False): 181 | """Given a list of tx hashes, store the trace data""" 182 | start = time.time() 183 | calls = [('debug_traceTransaction', [Tx, self.debug_traceTransactionSettings]) for Tx in Txs] 184 | results = self.batchRequests(calls) 185 | end = time.time() - start 186 | print("batch_getTrace time:", end) 187 | self.results = list(results) 188 | processNum = 20 189 | all_processes = [] 190 | for i in range(len(Txs)): 191 | process = multiprocessing.Process(target=self.cookStoreTrace2, args=(i, \ 192 | FullTrace, category, contract, Txs[i])) 193 | process.start() 194 | all_processes.append(process) 195 | if i != 0 and (i % processNum == 0 or i == len(Txs) - 1): 196 | for process in all_processes: 197 | process.join() 198 | all_processes = [] 199 | end2 = time.time() - (end + start) 200 | print("batch_storeTrace time:", end2) 201 | self.results = [] 202 | gc.collect() 203 | 204 | def batch_storeTrace(self, category: str, contract: str, Txs: list, FullTrace: bool = False): 205 | """Given a list of tx hashes, store the trace data""" 206 | start = time.time() 207 | calls = [('debug_traceTransaction', [Tx, self.debug_traceTransactionSettings]) for Tx in Txs] 208 | results = self.batchRequests(calls) 209 | end = time.time() - start 210 | print("batch_getTrace time:", end) 211 | processNum = 10 212 | all_processes = [] 213 | i = -1 214 | for result in results: 215 | i += 1 216 | process = multiprocessing.Process(target=self.cookStoreTrace, args=(result, \ 217 | FullTrace, category, contract, Txs[i])) 218 | process.start() 219 | all_processes.append(process) 220 | 221 | if i != 0 and (i % processNum == 0 or i == len(Txs) - 1): 222 | for process in all_processes: 223 | process.join() 224 | all_processes = [] 225 | end2 = time.time() - (end + start) 226 | print("batch_storeTrace time:", end2) 227 | 228 | def batch_getTrace(self, Txs: list, FullTrace: bool = False): 229 | """Given a list of tx hashes, return a list of trace data""" 230 | calls = [('debug_traceTransaction', [Tx, self.debug_traceTransactionSettings]) for Tx in Txs] 231 | results = self.batchRequests(calls) 232 | result_dicts = [self.cookResult(result, FullTrace=FullTrace) for result in results] 233 | return result_dicts 234 | 235 | def getTrace(self, Tx: str, FullTrace: bool = False): 236 | """Given a tx hash, return the trace data""" 237 | web3 = self.get_w3() 238 | start = time.time() 239 | gc.collect() 240 | print(Tx) 241 | result = None 242 | try: 243 | result = web3.manager.request_blocking('debug_traceTransaction', [Tx, self.debug_traceTransactionSettings]) 244 | except MemoryError: 245 | print("MemoryError when collecting trace data " + Tx, file=sys.stderr) 246 | 247 | end = time.time() - start 248 | print("Tx {} fetch trace costs {} s".format(Tx[0:4], end)) 249 | 250 | result_dict = self.cookResult(result, FullTrace=FullTrace) 251 | print("Tx {} cooking trace costs {} s".format(Tx[0:4], time.time() - start - end)) 252 | return result_dict 253 | 254 | def cookResult(self, result, FullTrace: bool = False): 255 | result_dict = dict(result) 256 | lastOpcode = {} # last opcode of the same depth 257 | for ii in range(len(result_dict['structLogs'])): 258 | structLog = result_dict['structLogs'][ii] 259 | structLog_dict = dict(structLog) 260 | depth = structLog_dict['depth'] 261 | if depth not in lastOpcode: 262 | structLog_dict_copy = self.pruneStructLog(structLog_dict, FullTrace=FullTrace) 263 | else: 264 | structLog_dict_copy = self.pruneStructLog(structLog_dict, lastOpcode[depth], FullTrace=FullTrace) 265 | result_dict['structLogs'][ii] = structLog_dict_copy 266 | lastOpcode[depth] = structLog_dict_copy["op"] 267 | return result_dict 268 | 269 | def storeTrace(self, txHash: str, FullTrace: bool = False): 270 | """Given a tx hash, store the trace data""" 271 | result_dict = self.getTrace(txHash, FullTrace=FullTrace) 272 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 273 | path = SCRIPT_DIR + '/../Benchmarks_Traces' + '/Txs/{}.json.gz'.format(txHash) 274 | writeCompressedJson(path, result_dict) 275 | 276 | 277 | 278 | def main(): 279 | fe = fetcher() 280 | # HackTx = "0x395675b56370a9f5fe8b32badfa80043f5291443bd6c8273900476880fb5221e" 281 | # fe.storeTrace("DeFiHackLabs", "0x051ebd717311350f1684f89335bed4abd083a2b6", HackTx, True ) 282 | pass 283 | 284 | 285 | 286 | 287 | if __name__ == "__main__": 288 | main() 289 | 290 | 291 | -------------------------------------------------------------------------------- /images/Comparison.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeffchen006/OpenTracer/6d3162c351051b82b1184aaaeb84034b0b1f959e/images/Comparison.png -------------------------------------------------------------------------------- /images/Overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeffchen006/OpenTracer/6d3162c351051b82b1184aaaeb84034b0b1f959e/images/Overview.png -------------------------------------------------------------------------------- /parserPackage/README.md: -------------------------------------------------------------------------------- 1 | ## Code Structure 2 | 3 | The code is structured as follows: 4 | 5 | * parser.py: a localized version of the parserGlobal. It only cares about one contract. 6 | 7 | Given a contract C, a list of Txs Txs. 8 | 9 | 10 | 11 | For each transaction 12 | 13 | * parser.py: The main parser class, basically does anything similar to a transaction viewer. Only cares about execution tree, call/staticcall/delegatecall/return/stop/gasless. It gives high-level info like the execution tree, call data, return value, msg.sender. 14 | 15 | * functions.py: Some helper functions for the parser class and the TraceTree class which is used to build the execution tree. 16 | 17 | * TraceTree.py: The TraceTree class which is used to build the execution tree. 18 | 19 | * tracker.py: the tracker class tracks the stack changes. And apply data flow analysis. 20 | 21 | 22 | -------------------------------------------------------------------------------- /parserPackage/dataSource.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 4 | sys.path.append(os.path.dirname(SCRIPT_DIR)) 5 | 6 | 7 | 8 | class dataSource: 9 | """It represents some end points data source""" 10 | def __init__(self, d: dict = None, opcode: str = None): 11 | self.sources = [] 12 | self.children = [] 13 | self.metaData = {} 14 | if d is not None: 15 | self.sources.append(d) 16 | self.children.append(None) 17 | if opcode is not None: 18 | self.sources.append(opcode) 19 | self.children.append(None) 20 | 21 | def to_dict(self): 22 | children = [child.to_dict() if child is not None else None for child in self.children] 23 | return {"sources": self.sources, "children": children, "metaData": self.metaData} 24 | 25 | def addChild(self, child): 26 | self.children.append(child) 27 | 28 | def isEmpty(self): 29 | return len(self.sources) == 0 30 | 31 | def addFunc(self, d: dict, child = None): 32 | isEqual = False 33 | for item in self.sources: 34 | if item == d: 35 | isEqual = True 36 | break 37 | if not isEqual: 38 | self.sources.append(d) 39 | self.children.append(child) 40 | 41 | def addOpcode(self, opcode: str, child = None): 42 | if opcode not in self.sources: 43 | self.sources.append(opcode) 44 | self.children.append(child) 45 | 46 | def merge(self, other): 47 | for ii in range(len(other.sources)): 48 | source = other.sources[ii] 49 | child = other.children[ii] 50 | if isinstance(source, dict): 51 | self.addFunc(source, child) 52 | else: 53 | self.addOpcode(source, child) 54 | 55 | def endPoints(self) -> list: 56 | endPoints = [] 57 | for ii in range(len(self.sources)): 58 | source = self.sources[ii] 59 | if self.children[ii] == None: 60 | endPoints.append(source) 61 | else: 62 | endPoints += self.children[ii].endPoints() 63 | return endPoints 64 | 65 | def __str__(self) -> str: 66 | string = "[" 67 | endPoints = self.endPoints() 68 | for ii in range(len(endPoints)): 69 | if ii != 0: 70 | string += ", " 71 | string += str(endPoints[ii]) 72 | string += "]" 73 | return string 74 | 75 | 76 | def getEndPoints(dataS: dict): 77 | endPoints = [] 78 | for ii in range(len(dataS["sources"])): 79 | source = dataS["sources"][ii] 80 | if dataS["children"][ii] == None: 81 | endPoints.append(source) 82 | else: 83 | endPoints += getEndPoints(dataS["children"][ii]) 84 | return endPoints -------------------------------------------------------------------------------- /parserPackage/decoder.py: -------------------------------------------------------------------------------- 1 | import struct 2 | import sys 3 | import os 4 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 5 | sys.path.append(os.path.dirname(SCRIPT_DIR)) 6 | 7 | import time 8 | import json 9 | from crawlPackage.crawlEtherscan import CrawlEtherscan 10 | from crawlPackage.crawlQuicknode import CrawlQuickNode 11 | from staticAnalyzer.analyzer import Analyzer 12 | from web3 import Web3 13 | import copy 14 | 15 | from eth_abi import decode, encode 16 | import eth_abi 17 | 18 | 19 | class decoder: 20 | def __init__(self) -> None: 21 | pass 22 | 23 | def formatCalldataArray(self, calldataArr: list) -> str: 24 | """Given a list of calldata, return a string of formatted calldata""" 25 | calldata = "" 26 | for calldataItem in calldataArr: 27 | calldata += self.formatCalldata(calldataItem) 28 | return calldata 29 | 30 | def formatCalldata(self, calldata: str) -> str: 31 | """Given a string of hex data, return a string of formatted calldata""" 32 | calldata = self.addPadding(calldata) 33 | return calldata.upper() 34 | 35 | def addPaddingUINT256(self, data: str) -> str: 36 | """Given a string of hex data, add padding to the front""" 37 | if data.startswith("0x"): 38 | data = data[2:] 39 | if len(data) % 64 != 0: 40 | data = '0' * (64 - len(data) % 64) + data 41 | return data 42 | 43 | def addPadding(self, data: str) -> str: 44 | """Given a string of hex data, add padding to the front""" 45 | if data.startswith("0x"): 46 | data = data[2:] 47 | if len(data) % 64 != 0: 48 | data = '0' * (64 - len(data) % 64) + data 49 | return data 50 | 51 | 52 | def getCalldataHex(self, oldCalldataHex: str, calldataSizeInt: int, startIndexHex: str, calldataRead: str) -> str: 53 | startIndex = int(startIndexHex, base = 16) 54 | return self.getCalldata(oldCalldataHex, calldataSizeInt, startIndex, calldataRead) 55 | 56 | 57 | def getCalldata(self, oldCalldataHex: str, calldataSize: int, startIndex: int, calldataRead: str) -> str: 58 | inputcalldataSize = calldataSize 59 | if calldataSize == -1: 60 | # special case: calldataSize is not known 61 | calldataSize = max(len(oldCalldataHex) // 2, startIndex + 32) 62 | 63 | if oldCalldataHex == "": 64 | oldCalldataHex = "0" * calldataSize * 2 65 | startIndexHex = startIndex * 2 # Compensate for '0x' 66 | calldataReadHex = self.formatCalldata(calldataRead) 67 | endIndexHex = startIndexHex + 64 68 | 69 | if endIndexHex > calldataSize * 2: 70 | s = oldCalldataHex[:startIndexHex] + calldataReadHex 71 | else: 72 | if endIndexHex > len(oldCalldataHex) and inputcalldataSize != -1: 73 | sys.exit("Decoder Error: endIndexHex > len(oldCalldataHex)") 74 | 75 | # if len(calldataReadHex) != 64: 76 | # sys.exit("Decoder Error: len(calldataReadHex) != 64") 77 | 78 | s = oldCalldataHex[:startIndexHex] + calldataReadHex + oldCalldataHex[endIndexHex:] 79 | return s 80 | 81 | 82 | 83 | 84 | def decodeSimpleABI(self, types: list, data: str) -> list: 85 | """Given a list of types and data, return a list of decoded data""" 86 | decodedData = decode(types, bytes.fromhex(data)) 87 | mylist = list(decodedData) 88 | for i in range(len(mylist)): 89 | if isinstance(mylist[i], bytes): 90 | mylist[i] = mylist[i].hex() 91 | return mylist 92 | 93 | 94 | def decodeReturn(self, types: list, memoryList: list, offset, length): 95 | """Given a list of types, memory (a list of str), an offset (eg. 0x100), a length (eg. 0x20), return decoded return value""" 96 | memorySnippet = self.extractMemory(memoryList, offset, length) 97 | decodedData = self.decodeSimpleABI(types, memorySnippet) 98 | return decodedData 99 | 100 | 101 | def extractMemory(self, memoryList: list, offset, length): 102 | """Given memory (a list of str), an offset (eg. 0x100), a length (eg. 0x20), return memory snippet which represents return value""" 103 | memoryWhole = ''.join(memoryList) 104 | offsetInt = int(offset, base = 16) 105 | lengthInt = int(length, base = 16) 106 | offsetBytesLen = offsetInt * 2 107 | lengthBytesLen = lengthInt * 2 108 | memorySnippet = memoryWhole[offsetBytesLen: offsetBytesLen + lengthBytesLen] 109 | # print("offsetBytesLen: ", offsetBytesLen) 110 | # print("offsetBytesLen + lengthBytesLen: ", offsetBytesLen + lengthBytesLen) 111 | # print("len(memoryWhole): ", len(memoryWhole)) 112 | # print(memorySnippet) 113 | return memorySnippet 114 | 115 | 116 | # uint bit 117 | typeLengths = { 118 | "uint": 32, 119 | "uint256": 32, 120 | "address": 20 + 12, # for padding 121 | } 122 | def get_memory_lengths(self, param_types, length): 123 | # Calculate the memory length for each parameter type 124 | # first decode and then encode, compared with the original length 125 | memory_lengths = [] 126 | for param_type in param_types: 127 | if param_type in self.typeLengths: 128 | memory_lengths.append(self.typeLengths[param_type]) 129 | else: 130 | sys.exit("decoder.py: Error: unknown type: " + param_type) 131 | 132 | # check if sum of memory lengths is equal to length 133 | if sum(memory_lengths) != length: 134 | sys.exit("decoder.py: Error: sum of memory lengths is not equal to length") 135 | 136 | return memory_lengths 137 | 138 | 139 | def get_padded_size(types): 140 | size = 0 141 | padded_size = 0 142 | for t in types: 143 | if t.startswith("uint") or t.startswith("int"): 144 | size += int(t[4:]) // 8 145 | elif t == "address": 146 | size += 20 147 | elif t.startswith("bytes"): 148 | size += int(t[5:]) 149 | else: 150 | raise ValueError(f"Unknown type: {t}") 151 | 152 | if size % 32 != 0: 153 | padded_size += (size // 32 + 1) * 32 154 | else: 155 | padded_size += size 156 | 157 | return (size, padded_size) 158 | 159 | 160 | 161 | if __name__ == "__main__": 162 | types = ['uint256'] 163 | # calldata = b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0009' 164 | # print(bytes.fromhex("B9A1")) 165 | # decodedData = decoder.decodeSimpleABI(types, calldata) 166 | 167 | decoder = decoder() 168 | # calldata = "0xc685fa11e01ec6f000000" 169 | # calldata = decoder.formatCalldata(calldata) 170 | # decodedData = decoder.decodeSimpleABI(types, calldata) 171 | # print(decodedData) 172 | 173 | # test extractMemory 174 | memory = ['0000000000000000000000000000000000000000000000000000000000000000', '0000000000000000000000000000000000000000000000000000000000000000', '0000000000000000000000000000000000000000000000000000000000000080', '0000000000000000000000000000000000000000000000000000000000000000', '0000000000000000000000000000000000000000047887633ebc2f527e747ec1'] 175 | # Function Returns memory[0x80:0x80+0x20] 176 | snippet = decoder.extractMemory(memory, "0x80", "0x20") 177 | 178 | types = ['uint256'] 179 | decodedData = decoder.decodeSimpleABI(types, snippet) 180 | print(decodedData) 181 | 182 | types = ['address'] 183 | data = "000000000000000000000000BFCF63294AD7105DEA65AA58F8AE5BE2D9D0952A" 184 | ret = decode(types, bytes.fromhex(data)) 185 | print(ret) -------------------------------------------------------------------------------- /parserPackage/functions.py: -------------------------------------------------------------------------------- 1 | import json 2 | import sys 3 | import os 4 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 5 | sys.path.append(os.path.dirname(SCRIPT_DIR)) 6 | 7 | def addLeadningZeroFuncSelector(funcSelector: str) -> str: 8 | """Given a function selector, add leading zeros to make it 5 bytes""" 9 | return funcSelector[0:2] + "0" * (10 - len(funcSelector)) + funcSelector[2:] 10 | 11 | def getTrace(jsonFile: str): 12 | """Given a transaction hash, return the trace""" 13 | content = None 14 | with open(jsonFile, 'r') as f: 15 | content = json.load(f) 16 | return content 17 | 18 | def getPathFromCategoryTxHash(category: str, contract: str, txHash: str): 19 | """Given a category, contract and txHash, return the path""" 20 | path = SCRIPT_DIR + '/../Benchmarks_Traces' + '/{}/Txs/{}/{}.json.gz'.format(category, contract, txHash) 21 | return path 22 | 23 | 24 | 25 | 26 | class stackEntry: 27 | # Do we really need to reason about different bytes? 28 | # 00000000000000000000000000000000000 29 | # ------------- 30 | # (1) ------------- 31 | # (2) --------- 32 | # (3) 33 | # Like, reason different bytes 34 | def __init__(self, dataSrcInfo) -> None: 35 | self.asWhole = True 36 | self.dataSrcInfo = dataSrcInfo 37 | 38 | def __init__(self, separators, dataSrcInfos, size) -> None: 39 | self.asWhole = False 40 | self.separators = separators 41 | self.dataSrcInfos = dataSrcInfos 42 | self.size = size 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /parserPackage/locator.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 4 | sys.path.append(os.path.dirname(SCRIPT_DIR)) 5 | 6 | # a list of types the tracker could track 7 | EVENT = 0 8 | FUNCTION = 1 9 | RETURNVALUE = 2 10 | SELFCALLVALUE = 3 # call value attached to the function call 11 | CALLVALUE = 4 # call value attached to an external function call 12 | FALLBACK = 5 # call value attached to a fallback function call 13 | 14 | # physical meaning 15 | DEPOSIT = 99 16 | INVEST = 100 17 | WITHDRAW = 101 18 | TRANSFER = 102 19 | 20 | # # Special address 21 | # MSGSENDER = 555 22 | # TXORIGIN = 556 23 | 24 | 25 | # class locator marks a list of locations which could be manipulated 26 | class locator: 27 | def __init__(self, targetFunc, type, *, name = None, funcPara = [], \ 28 | funcAddress = None, position = None, returnValuePosition = None, \ 29 | fromAddr = None): 30 | '''funcPara means the value of function's another parameter''' 31 | self.type = type 32 | self.targetFunc = targetFunc 33 | self.fromAddr = fromAddr 34 | if type == FUNCTION: 35 | self.funcName = name 36 | self.argPosition = position 37 | self.funcPara = funcPara 38 | self.funcAddress = funcAddress 39 | elif type == EVENT: 40 | self.eventName = name 41 | self.eventPosition = position 42 | elif type == RETURNVALUE: 43 | self.returnValuePosition = returnValuePosition 44 | elif type == SELFCALLVALUE: 45 | pass 46 | elif type == CALLVALUE: 47 | self.funcAddress = funcAddress 48 | pass 49 | elif type == FALLBACK: 50 | pass 51 | self.kind = None 52 | 53 | 54 | def isTrackedDeeper(self): 55 | if self.type == FUNCTION: 56 | return True 57 | else: 58 | return False 59 | 60 | 61 | 62 | if __name__ == "__main__": 63 | l1 = locator("func1", FUNCTION, funcPara = (1, "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2"), position=2) 64 | l2 = locator("event1", EVENT, position=1) 65 | 66 | -------------------------------------------------------------------------------- /settings.toml: -------------------------------------------------------------------------------- 1 | [version] 2 | current = "v0.0.0-alpha" 3 | 4 | [settings] 5 | EtherScanApiKeys = ["MBXWI4I69KTZW873HZHKG6DDRGHGW4B1EM", "GA7PY4PE6I3PIYXDSMWW4I98QASKSA11DK", "THF8Q6FVQCYSSHUBY89RX987IFEW5FBYD7"] # free etherScan API keys can be got at https://etherscan.io/myapikey 6 | rpcProviders = [ "https://docs-demo.quiknode.pro/" ] # this is a demo rpc provider by QuickNode, for large scale use, you can get a paid plan at https://www.quiknode.io/ 7 | ethArchives = [ "https://eth.llamarpc.com" ] # free rpc providers can also be got at alchemy.com 8 | 9 | [runtime] 10 | LoggingUpperBound = 1000 11 | 12 | [experiment] 13 | trainingSetRatio = 0.7 14 | -------------------------------------------------------------------------------- /staticAnalyzer/README.md: -------------------------------------------------------------------------------- 1 | This is a static analyzer which gets all information from smart contracts(alternatively, verified contract address). 2 | 3 | It includes wrappers to get information from Solidity and Vyper compilers. 4 | 5 | It also contains a wrapper of Slither to better support this project, cross-validating results from Solidity and Vyper compilers. 6 | 7 | 8 | 9 | Particularly, for contracts compiled with vyper-0.2.15 or before, there is no such an "-f layout" option, so I really cannot find storage layout. 10 | 11 | Therefore, it requires me to predict storage Layout using past trace data. Luckily, it is not a common case and I can do it manually. 12 | 13 | 14 | Other scenarios include developers used their customized storage Layout file (Vyper) or even customized compilers to compiler the contracts. 15 | 16 | 17 | Recently, a couple of decompilers are open sourced. They can be used to predict storage layout for unverified contracts, interested readers can check it out: 18 | 19 | https://github.com/smlxl/storage-layout-extractor 20 | 21 | https://github.com/nevillegrech/gigahorse-toolchain 22 | 23 | https://github.com/banteg/storage-layout 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /staticAnalyzer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeffchen006/OpenTracer/6d3162c351051b82b1184aaaeb84034b0b1f959e/staticAnalyzer/__init__.py -------------------------------------------------------------------------------- /staticAnalyzer/analyzer.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 4 | sys.path.append(os.path.dirname(SCRIPT_DIR)) 5 | 6 | from crawlPackage.crawlEtherscan import CrawlEtherscan 7 | from crawlPackage.crawlQuicknode import CrawlQuickNode 8 | # from crawlPackage.crawlTrueBlocks import CrawlTrueBlocks 9 | from staticAnalyzer.slitherAnalyzer import slitherAnalyzer 10 | from staticAnalyzer.vyperAnalyzer import vyperAnalyzer 11 | 12 | import subprocess 13 | import json 14 | import time 15 | import random 16 | import pickle 17 | import copy 18 | import sqlite3 19 | from crawlPackage.cacheDatabase import _save_transaction_receipt, _load_transaction_receipt, _save_contract, _load_contract 20 | 21 | 22 | 23 | # Solidity starts from 0.4.11 to 0.8.17 24 | # Vyper starts from 0.1.0-beta.16 to 0.3.7 25 | # Vyper starts to support storage layout option from 0.2.16 26 | 27 | 28 | def save_object(obj, filename: str): 29 | # print("filename: ", filename) 30 | try: 31 | with open(SCRIPT_DIR + "/cache/" + "{}.pickle".format(filename), "wb") as f: 32 | pickle.dump(obj, f, protocol=pickle.HIGHEST_PROTOCOL) 33 | except Exception as ex: 34 | print("Error during pickling object (Possibly unsupported):", ex) 35 | finally: 36 | pass 37 | 38 | 39 | def load_object(filename: str): 40 | # print("read filename: ", filename) 41 | try: 42 | with open(SCRIPT_DIR + "/cache/" + "{}.pickle".format(filename), "rb") as f: 43 | value = pickle.load(f) 44 | return value 45 | except Exception as ex: 46 | return None 47 | finally: 48 | pass 49 | 50 | 51 | 52 | class Analyzer: 53 | 54 | 55 | def __init__(self) -> None: 56 | self.crawlEtherscan = CrawlEtherscan() 57 | self.slitherAnalyzer = slitherAnalyzer() 58 | self.vyperAnalyzer = vyperAnalyzer() 59 | self.isVyperCache = {} 60 | self.storageMappingMapping = {} 61 | self.funcSigMapMapping = {} 62 | self.unableCompileAddresses = [] 63 | 64 | SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) 65 | etherScanDatabase = SCRIPT_DIR + "/../crawlPackage/database/etherScan.db" 66 | if os.path.exists(etherScanDatabase): 67 | self.conn = sqlite3.connect(etherScanDatabase) 68 | self.cur = self.conn.cursor() 69 | 70 | self.slitherAnalyzer.cur = self.cur 71 | self.vyperAnalyzer.cur = self.cur 72 | 73 | 74 | 75 | # def contract2storageLayout(self, contractAddress: str) -> dict: 76 | # """Given a contract address, return a map ( -> )""" 77 | # filename = "{}_storageLayout".format(contractAddress.lower()) 78 | # storage_layout = load_object(filename) 79 | # if storage_layout is not None: 80 | # return storage_layout 81 | # if self.isVyper(contractAddress): 82 | # storage_layout = self.vyperAnalyzer.Contract2storageLayout(contractAddress) 83 | # else: 84 | # storage_layout = self.slitherAnalyzer.Contract2storageLayout(contractAddress) 85 | # if storage_layout == None: 86 | # sys.exit("Error: cannot read storage layout of {}!".format(contractAddress)) 87 | 88 | # save_object(storage_layout, filename) 89 | # return storage_layout 90 | 91 | def contract2storageMapping(self, contractAddress: str) -> dict: 92 | """Given a contract, return a map ( -> )""" 93 | if contractAddress in self.storageMappingMapping: 94 | return self.storageMappingMapping[contractAddress] 95 | 96 | if contractAddress in self.unableCompileAddresses: 97 | return None 98 | 99 | anotherFileName = "UnableCompileAddresses" 100 | UnableCompileAddresses = load_object(anotherFileName) 101 | if UnableCompileAddresses is None: 102 | self.unableCompileAddresses = [] 103 | else: 104 | self.unableCompileAddresses = UnableCompileAddresses 105 | 106 | filename = "{}_storageMapping".format(contractAddress.lower()) 107 | storage_mapping = load_object(filename) 108 | if storage_mapping is not None and storage_mapping != {}: 109 | self.storageMappingMapping[contractAddress] = storage_mapping 110 | return storage_mapping 111 | 112 | try: 113 | if not self.isVyper(contractAddress): 114 | storage_mapping = self.slitherAnalyzer.Contract2storageMapping(contractAddress) 115 | self.storageMappingMapping[contractAddress] = storage_mapping 116 | else: 117 | storage_mapping = self.vyperAnalyzer.Contract2storageMapping(contractAddress) 118 | self.storageMappingMapping[contractAddress] = storage_mapping 119 | except: 120 | storage_mapping = None 121 | 122 | if storage_mapping == None: 123 | # print("Error: cannot read storage mapping of {}!".format(contractAddress)) 124 | # print("possible reason: the contract is written in Vyper < 0.2.16") 125 | self.unableCompileAddresses.append(contractAddress) 126 | save_object(self.unableCompileAddresses, anotherFileName) 127 | pass 128 | save_object(storage_mapping, filename) 129 | return storage_mapping 130 | 131 | def imple2funcSigMap(self, contractAddress: str, implementationAddress: str): 132 | filename = "{}_funcSigMap".format(contractAddress.lower()) 133 | funcSigMap2 = load_object(filename) 134 | 135 | funcSigMap = self.contract2funcSigMap(implementationAddress) 136 | for funcSelector in funcSigMap: 137 | if funcSelector not in funcSigMap2: 138 | funcSigMap2[funcSelector] = funcSigMap[funcSelector] 139 | save_object(funcSigMap2, filename) 140 | # print(funcSigMap2) 141 | return funcSigMap2 142 | 143 | 144 | 145 | def contract2funcSigMap(self, contractAddress: str): 146 | """Given a contract address, return a list of function selectors""" 147 | """{selector: (name, input_types, output_types)}""" 148 | """eg. {'0x771602f7': ('add', ['uint256', 'uint256'], ['uint256'], readOnly?)...}""" 149 | if contractAddress in self.funcSigMapMapping: 150 | return self.funcSigMapMapping[contractAddress] 151 | 152 | filename = "{}_funcSigMap".format(contractAddress.lower()) 153 | funcSigMap2 = load_object(filename) 154 | if funcSigMap2 is not None: 155 | self.funcSigMapMapping[contractAddress] = funcSigMap2 156 | return funcSigMap2 157 | 158 | anotherFileName = "UnverifiedAddresses" 159 | UnverifiedAddresses = load_object(anotherFileName) 160 | if UnverifiedAddresses is None: 161 | self.UnverifiedAddresses = [] 162 | else: 163 | self.UnverifiedAddresses = UnverifiedAddresses 164 | 165 | if contractAddress in self.UnverifiedAddresses: 166 | return {} 167 | 168 | # func sig map from public ABI 169 | funcSigMap = self.crawlEtherscan.Contract2funcSigMap2(contractAddress) 170 | if len(funcSigMap.keys()) == 0: # means the contract is not verified on etherscan 171 | self.funcSigMapMapping[contractAddress] = {} 172 | self.UnverifiedAddresses.append(contractAddress) 173 | save_object(self.UnverifiedAddresses, anotherFileName) 174 | return {} 175 | 176 | funcSigMap2 = None 177 | try: 178 | self.crawlEtherscan.Contract2Sourcecode(contractAddress) 179 | if not self.isVyper(contractAddress): 180 | # func sig map from slither 181 | funcSigMap2 = self.slitherAnalyzer.Contract2funcSigMap(contractAddress) 182 | else: 183 | # func sig map from vyper compile results 184 | funcSigMap2 = self.vyperAnalyzer.Contract2funcSigMap(contractAddress) 185 | except Exception as ex: 186 | return funcSigMap 187 | 188 | # merge two maps 189 | for funcSelector in funcSigMap: 190 | # means it is a read-only function, in every case, it's a public variable query function. 191 | if funcSelector not in funcSigMap2: 192 | funcSigMap2[funcSelector] = funcSigMap[funcSelector] 193 | 194 | switch_filename = "switchMap" 195 | switchMap = load_object(switch_filename) 196 | if switchMap is None: 197 | switchMap = {} 198 | keys = list(switchMap.keys()) 199 | for key in keys: 200 | if "0x" in key: 201 | switchMap.pop(key) 202 | 203 | # convert non primitive types to address 204 | for funcSelector in funcSigMap2: 205 | for jj in [1, 2]: 206 | if jj == 2 and funcSelector == "constructor": 207 | continue 208 | for ii in range(len(funcSigMap2[funcSelector][jj])): 209 | if funcSigMap2[funcSelector][jj][ii] in switchMap: 210 | funcSigMap2[funcSelector][jj][ii] = switchMap[ funcSigMap2[funcSelector][jj][ii] ] 211 | 212 | 213 | # check if there is a function selector collision 214 | for funcSelector in funcSigMap: 215 | if funcSelector == "constructor": 216 | continue 217 | if funcSelector in funcSigMap2: 218 | if ( len(funcSigMap[funcSelector]) != len(funcSigMap2[funcSelector]) \ 219 | or len(funcSigMap[funcSelector][1]) != len(funcSigMap2[funcSelector][1]) \ 220 | or len(funcSigMap[funcSelector][2]) != len(funcSigMap2[funcSelector][2]) \ 221 | or funcSigMap[funcSelector][0] != funcSigMap2[funcSelector][0] ): 222 | 223 | funcSigMap2[funcSelector] = funcSigMap[funcSelector] 224 | # print(funcSigMap[funcSelector]) 225 | # print(funcSigMap2[funcSelector]) 226 | # sys.exit("Error: function selector different lengths !") 227 | 228 | for ii in [1, 2]: 229 | for jj in range(len(funcSigMap[funcSelector][ii])): 230 | if funcSigMap[funcSelector][ii][jj] != funcSigMap2[funcSelector][ii][jj]: 231 | switchMap[ copy.deepcopy(funcSigMap2[funcSelector][ii][jj]) ] = funcSigMap[funcSelector][ii][jj] 232 | funcSigMap2[funcSelector][ii][jj] = funcSigMap[funcSelector][ii][jj] 233 | 234 | 235 | self.funcSigMapMapping[contractAddress] = funcSigMap2 236 | save_object(switchMap, switch_filename) 237 | save_object(funcSigMap2, filename) 238 | 239 | return funcSigMap2 240 | 241 | 242 | def contract2funcSelectors(self, contractAddress: str) -> list: 243 | """Given a contract address, return a list of function selectors""" 244 | funcSigMap = self.contract2funcSigMap(contractAddress) 245 | funcSelectors = list(funcSigMap.keys()) 246 | print(funcSelectors) 247 | return funcSelectors 248 | 249 | def isVyper(self, contractAddress: str): 250 | """Given a contract address, return True if the contract is Vyper""" 251 | if contractAddress in self.isVyperCache: 252 | return self.isVyperCache[contractAddress] 253 | 254 | CompilerVersion, _ = self.vyperAnalyzer.contract2Sourcecode(contractAddress) 255 | if CompilerVersion is None or (not CompilerVersion.startswith("vyper")): 256 | self.isVyperCache[contractAddress] = False 257 | return False 258 | self.isVyperCache[contractAddress] = True 259 | return True 260 | 261 | 262 | 263 | 264 | 265 | if __name__ == "__main__": 266 | analyzer = Analyzer() 267 | # analyzer.contract2storageMapping("0x15fda9f60310d09fea54e3c99d1197dff5107248") 268 | 269 | # analyzer.imple2funcSigMap("0x2069043d7556b1207a505eb459d18d908df29b55", "0xc68bf77e33f1df59d8247dd564da4c8c81519db6") 270 | # analyzer.imple2funcSigMap("0x44fbebd2f576670a6c33f6fc0b00aa8c5753b322", "0x3c710b981f5ef28da1807ce7ed3f2a28580e0754") 271 | 272 | contract = "0x2320a28f52334d62622cc2eafa15de55f9987ed9" 273 | 274 | # contract = "0xD533a949740bb3306d119CC777fa900bA034cd52" 275 | funcMap = analyzer.contract2funcSigMap(contract) 276 | print(funcMap) 277 | 278 | # # EMN contract 279 | # contractAddress = "0x5ade7aE8660293F2ebfcEfaba91d141d72d221e8" 280 | 281 | # # Yearn contract 282 | # contractAddress = "0xacd43e627e64355f1861cec6d3a6688b31a6f952" 283 | # # KeyError: '0xf8897945' 284 | 285 | # funcSigMap = analyzer.contract2funcSigMap(contractAddress) 286 | 287 | # # StrategyDAI3pool 288 | # contractAddress = "0x9c211bfa6dc329c5e757a223fb72f5481d676dc1" 289 | # funcSigMap = analyzer.contract2funcSigMap(contractAddress) 290 | 291 | # # test isVyper 292 | # # EMN contract 293 | # contractAddress = "0x5ade7aE8660293F2ebfcEfaba91d141d72d221e8" 294 | # print(analyzer.isVyper(contractAddress)) 295 | 296 | # contractAddress = "0x6b175474e89094c44da98b954eedeac495271d0f" 297 | # funcSigMap = analyzer.contract2funcSigMap(contractAddress) 298 | # import pprint 299 | # pp = pprint.PrettyPrinter(indent=4) 300 | # pp.pprint(funcSigMap) 301 | 302 | 303 | # # test contract2storageMapping 304 | # contractAddress = "0xd77e28a1b9a9cfe1fc2eee70e391c05d25853cbf" 305 | # storageMapping = analyzer.contract2storageMapping(contractAddress) 306 | # print(storageMapping) 307 | 308 | 309 | # ('0x85ca13d8496b2d22d6...1e096dd7e0', 'Mapping', '00000000000000000000...0000000019', 'CALLER', '0x195f9f44489b43e04', 5527) 310 | 311 | 312 | 313 | 314 | 315 | 316 | 317 | 318 | -------------------------------------------------------------------------------- /staticAnalyzer/cache/0x39aa39c021dfbae8fac545936693ac917d5e7563_storageMapping.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeffchen006/OpenTracer/6d3162c351051b82b1184aaaeb84034b0b1f959e/staticAnalyzer/cache/0x39aa39c021dfbae8fac545936693ac917d5e7563_storageMapping.pickle -------------------------------------------------------------------------------- /staticAnalyzer/cache/0x3bc6aa2d25313ad794b2d67f83f21d341cc3f5fb_storageMapping.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeffchen006/OpenTracer/6d3162c351051b82b1184aaaeb84034b0b1f959e/staticAnalyzer/cache/0x3bc6aa2d25313ad794b2d67f83f21d341cc3f5fb_storageMapping.pickle -------------------------------------------------------------------------------- /staticAnalyzer/cache/0x3d9819210a31b4961b30ef54be2aed79b9c9cd3b_storageMapping.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeffchen006/OpenTracer/6d3162c351051b82b1184aaaeb84034b0b1f959e/staticAnalyzer/cache/0x3d9819210a31b4961b30ef54be2aed79b9c9cd3b_storageMapping.pickle -------------------------------------------------------------------------------- /staticAnalyzer/cache/0x75442ac771a7243433e033f3f8eab2631e22938f_storageMapping.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeffchen006/OpenTracer/6d3162c351051b82b1184aaaeb84034b0b1f959e/staticAnalyzer/cache/0x75442ac771a7243433e033f3f8eab2631e22938f_storageMapping.pickle -------------------------------------------------------------------------------- /staticAnalyzer/cache/0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48_storageMapping.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeffchen006/OpenTracer/6d3162c351051b82b1184aaaeb84034b0b1f959e/staticAnalyzer/cache/0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48_storageMapping.pickle -------------------------------------------------------------------------------- /staticAnalyzer/cache/0xa2327a938febf5fec13bacfb16ae10ecbc4cbdcf_storageMapping.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeffchen006/OpenTracer/6d3162c351051b82b1184aaaeb84034b0b1f959e/staticAnalyzer/cache/0xa2327a938febf5fec13bacfb16ae10ecbc4cbdcf_storageMapping.pickle -------------------------------------------------------------------------------- /staticAnalyzer/cache/0xb650eb28d35691dd1bd481325d40e65273844f9b_storageMapping.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeffchen006/OpenTracer/6d3162c351051b82b1184aaaeb84034b0b1f959e/staticAnalyzer/cache/0xb650eb28d35691dd1bd481325d40e65273844f9b_storageMapping.pickle -------------------------------------------------------------------------------- /staticAnalyzer/cache/0xd8ec56013ea119e7181d231e5048f90fbbe753c0_storageMapping.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeffchen006/OpenTracer/6d3162c351051b82b1184aaaeb84034b0b1f959e/staticAnalyzer/cache/0xd8ec56013ea119e7181d231e5048f90fbbe753c0_storageMapping.pickle -------------------------------------------------------------------------------- /staticAnalyzer/slitherAnalyzer.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 4 | sys.path.append(os.path.dirname(SCRIPT_DIR)) 5 | 6 | from crawlPackage.crawlEtherscan import CrawlEtherscan 7 | # from crawlPackage.crawlQuicknode import CrawlQuickNode 8 | # from crawlPackage.crawlTrueBlocks import CrawlTrueBlocks 9 | from web3 import Web3, HTTPProvider 10 | 11 | from slither.slither import Slither 12 | from slither.core.solidity_types.elementary_type import ElementaryType 13 | from slither.core.solidity_types.user_defined_type import UserDefinedType 14 | from slither.core.solidity_types.mapping_type import MappingType 15 | from slither.core.solidity_types.array_type import ArrayType 16 | from slither.core.solidity_types.function_type import FunctionType 17 | 18 | 19 | import pickle 20 | import math 21 | import copy 22 | import toml 23 | settings = toml.load("settings.toml") 24 | 25 | 26 | 27 | 28 | 29 | """This is a wapper class for slither""" 30 | class slitherAnalyzer: 31 | def __init__(self) -> None: 32 | self.etherScan = CrawlEtherscan() 33 | self.cur = None 34 | 35 | 36 | def Contract2storageLayout(self, contractAddress: str) -> dict: 37 | """Given a contract address, return a map ( -> )""" 38 | if not self.etherScan.isVerified(contractAddress): 39 | return {} 40 | slither = Slither(contractAddress, **{"etherscan_api_key": self.etherScan.getEtherScanAPIkey()} ) 41 | compilation_uints = slither.compilation_units 42 | if len(compilation_uints) != 1: 43 | sys.exit("Error: multiple compilation units!") 44 | compilation_unit = compilation_uints[0] 45 | # Compared with compute_storage_layout() from compilation_unit 46 | # storage_layout2 also includes the constant and immutable variables 47 | storage_layout = {} 48 | for contract in compilation_unit.contracts_derived: 49 | storage_layout[contract.name] = {} 50 | slot = 0 51 | offset = 0 52 | for var in contract.state_variables_ordered: 53 | _type = var.type 54 | size, new_slot = var.type.storage_size 55 | if new_slot: 56 | if offset > 0: 57 | slot += 1 58 | offset = 0 59 | elif size + offset > 32: 60 | slot += 1 61 | offset = 0 62 | storage_layout[contract.name][var.canonical_name] = ( 63 | slot, 64 | offset, 65 | _type 66 | ) 67 | if new_slot: 68 | slot += math.ceil(size / 32) 69 | else: 70 | offset += size 71 | return storage_layout 72 | 73 | def Contract2storageMapping(self, contractAddress: str) -> dict: 74 | """Given a contract address, return a map ( -> )""" 75 | storageLayout = self.Contract2storageLayout(contractAddress) 76 | storageMapping = self.StorageLayout2StorageMapping(storageLayout) 77 | return storageMapping 78 | 79 | 80 | def parseType(self, _type): 81 | _type_stored = None 82 | if isinstance(_type, ElementaryType): 83 | _type_stored = _type._type 84 | elif isinstance(_type, UserDefinedType): 85 | # _type_stored = self.parseType(_type._type) 86 | _type_stored = str(_type) 87 | elif isinstance(_type, MappingType): 88 | _type_stored = (self.parseType(_type._from), self.parseType(_type._to)) 89 | elif isinstance(_type, ArrayType): 90 | # print attributes of the array type 91 | # print("meet arrat type") 92 | if _type._length is None: 93 | _type_stored = self.parseType(_type._type) + "[]" 94 | else: 95 | _type_stored = self.parseType(_type._type) + "[" + str(_type._length_value) + "]" 96 | 97 | elif isinstance(_type, FunctionType): 98 | sys.exit("Error! Cannot handle function type for now!") 99 | 100 | return _type_stored 101 | 102 | 103 | def StorageLayout2StorageMapping(self, storageLayout: dict) -> dict: 104 | """Given a storage layout, return a map ( -> )""" 105 | storageMapping = {} 106 | for contractName in storageLayout: 107 | for varName in storageLayout[contractName]: 108 | slot, offset, _type = storageLayout[contractName][varName] 109 | byte = slot * 32 + offset 110 | if byte not in storageMapping: 111 | """Ideally, _type_stored should be something parsable by eth_abi""" 112 | _type_stored = self.parseType(_type) 113 | storageMapping[byte] = (varName, _type_stored) 114 | return storageMapping 115 | 116 | def Contract2funcSigMap(self, contractAddress: str) -> dict: 117 | """Given a contract address, return a map ( -> )""" 118 | 119 | slither = Slither(contractAddress, **{"etherscan_api_key": self.etherScan.getEtherScanAPIkey()} ) 120 | funcSigMap = {} 121 | for contract in slither.contracts_derived: 122 | for function in contract.functions: 123 | funcSig = function.signature 124 | # eg. ('rely', ['address'], []) 125 | funcSigTuple = None 126 | if function.view or function.pure: 127 | funcSigTuple = (funcSig[0], funcSig[1], funcSig[2], True) 128 | else: 129 | funcSigTuple = (funcSig[0], funcSig[1], funcSig[2], False) 130 | funcSigStr = function.solidity_signature 131 | # eg. 'rely(address)' 132 | if funcSig[0] == "constructor": 133 | funcSelector = "constructor" 134 | else: 135 | funcSelector = Web3.keccak(text=funcSigStr).hex()[0:10] 136 | funcSigMap[funcSelector] = funcSigTuple 137 | 138 | return funcSigMap 139 | 140 | def contract2storageLayout_solc(self, contractAddress: str) -> dict: 141 | """Given a contract address, return a map ( -> )""" 142 | storageLayout = self.slitherAnalyzer.Contract2storageLayout(contractAddress) 143 | return storageLayout 144 | 145 | 146 | 147 | 148 | 149 | if __name__ == "__main__": 150 | slitherAnalyzer = slitherAnalyzer() 151 | # # test Contract2funcSigMap 152 | contractAddress = "0x5ade7aE8660293F2ebfcEfaba91d141d72d221e8" 153 | 154 | # Yearn contract 155 | contractAddress = "0xacd43e627e64355f1861cec6d3a6688b31a6f952" 156 | 157 | # StrategyDAI3pool 158 | contractAddress = "0x9c211bfa6dc329c5e757a223fb72f5481d676dc1" 159 | 160 | # DAI 161 | contractAddress = "0x6b175474e89094c44da98b954eedeac495271d0f" 162 | 163 | funcSigMap = slitherAnalyzer.Contract2funcSigMap(contractAddress) 164 | import pprint 165 | pp = pprint.PrettyPrinter(indent=4) 166 | pp.pprint(funcSigMap) 167 | 168 | # print("=====================================") 169 | 170 | # test Contract2storageLayout 171 | storageLayout = slitherAnalyzer.Contract2storageLayout(contractAddress) 172 | print(storageLayout) 173 | 174 | # print("=====================================") 175 | 176 | storageMapping = slitherAnalyzer.StorageLayout2StorageMapping(storageLayout) 177 | print(storageMapping) 178 | 179 | # Test check if a contract is written in Vyper or not 180 | 181 | # UniswapAddress = "0x1f9840a85d5aF5bf1D1762F925BDADdC4201F984" 182 | # slither = Slither(contractAddress, **{"etherscan_api_key": CrawlEtherscan().getEtherScanAPIkey()} ) 183 | # # Test storage layout 184 | # storageMapping = slitherAnalyzer.Contract2storageMapping("0x9e65ad11b299ca0abefc2799ddb6314ef2d91080") 185 | # print(storageMapping) 186 | 187 | # # {0: ('StrategyDAI3pool.want', 'address'), 188 | # # 32: ('StrategyDAI3pool._3pool', 'address'), 189 | # # 64: ('StrategyDAI3pool._3crv', 'address'), 190 | # # 96: ('StrategyDAI3pool.y3crv', 'address'), 191 | # # 128: ('StrategyDAI3pool.ypool', 'address'), 192 | # # 160: ('StrategyDAI3pool.ycrv', 'address'), 193 | # # 192: ('StrategyDAI3pool.yycrv', 'address'), 194 | # # 224: ('StrategyDAI3pool.dai', 'address'), 195 | # # 256: ('StrategyDAI3pool.ydai', 'address'), 196 | # # 288: ('StrategyDAI3pool.usdc', 'address'), 197 | # # 320: ('StrategyDAI3pool.yusdc', 'address'), 198 | # # 352: ('StrategyDAI3pool.usdt', 'address'), 199 | # # 384: ('StrategyDAI3pool.yusdt', 'address'), 200 | # # 416: ('StrategyDAI3pool.tusd', 'address'), 201 | # # 448: ('StrategyDAI3pool.ytusd', 'address'), 202 | # # 480: ('StrategyDAI3pool.governance', 'address'), 203 | # # 512: ('StrategyDAI3pool.controller', 'address'), 204 | # # 544: ('StrategyDAI3pool.strategist', 'address'), 205 | # # 576: ('StrategyDAI3pool.DENOMINATOR', 'uint256'), 206 | # # 608: ('StrategyDAI3pool.treasuryFee', 'uint256'), 207 | # # 640: ('StrategyDAI3pool.withdrawalFee', 'uint256'), 208 | # # 672: ('StrategyDAI3pool.strategistReward', 'uint256'), 209 | # # 704: ('StrategyDAI3pool.threshold', 'uint256'), 210 | # # 736: ('StrategyDAI3pool.slip', 'uint256'), 211 | # # 768: ('StrategyDAI3pool.tank', 'uint256'), 212 | # # 800: ('StrategyDAI3pool.p', 'uint256'), 213 | # # 832: ('StrategyDAI3pool.flag', 'bool')} 214 | -------------------------------------------------------------------------------- /staticAnalyzer/storageMappings/README.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | This folder contains the storage mappings for some contracts which cannot be analyzed using compiler outputs. 4 | 5 | Particularly, for contracts compiled with vyper-0.2.15 or before, there is no such an "-f layout" option, so I really cannot find storage layout. 6 | 7 | Therefore, it requires me to predict storage Layout using past trace data. Luckily, it is not a common case and I can do it manually. 8 | 9 | 10 | Other scenarios include developers used their customized storage Layout file (Vyper) or even customized compilers to compiler the contracts. 11 | 12 | 13 | Recently, a couple of decompilers are open sourced. They can be used to predict storage layout for unverified contracts, interested readers can check it out: 14 | 15 | https://github.com/smlxl/storage-layout-extractor 16 | 17 | https://github.com/nevillegrech/gigahorse-toolchain 18 | 19 | https://github.com/banteg/storage-layout 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /staticAnalyzer/temp.txt: -------------------------------------------------------------------------------- 1 | https://eth.llamarpc.com 2 | https://eth-mainnet.g.alchemy.com/v2/weftbAqPky4MPT-_G1v9NNofoctXc_Mi 3 | Time elapsed for collecting transaction history: 2.210395097732544 s 4 | Time elapsed for downloading transaction traces: 0.023702144622802734 s 5 | Time elapsed for parsing transaction traces: 7.960000038146973 s 6 | Time elapsed for analyzing tx: 0x39a78785a85250ee6f17459113efa2bdc2d5069a37f751171ee44efb3ac219f7 1.971790075302124 s 7 | Time elapsed for analyzing tx: 0x1f805ec425c94fa408bac529734ca84f7ba2bdc5df164e8f60a797a40d40b77f 1.1898088455200195 s 8 | Time elapsed for analyzing tx: 0x8c7cc6ab0a5ed76098152927c52a7f3d3ded6f95783fbd13e05b174574bb2763 1.124871015548706 s 9 | Time elapsed for analyzing tx: 0x4d55f5c95fe23153c00443125842cbeac2d06c4b550bba0539f05c15aaa2a19a 3.6804418563842773 s 10 | now we arrive at the function call transfer, ii = 15129 11 | now we arrive at the function call transfer, ii = 23926 12 | Time elapsed for analyzing tx: 0x3f5916f688847f31e0714aabe5bef946b4130d859d130260a52e7b220f752c56 4.585702180862427 s 13 | Time elapsed for analyzing tx: 0xa375d7d05d40fa2dad8ca279610a5b3f48e391f9e4aa6e31de5f2a4d3d482858 3.391124963760376 s 14 | now we arrive at the function call transfer, ii = 16520 15 | now we arrive at the function call transfer, ii = 25317 16 | Time elapsed for analyzing tx: 0xa64b5631a70227a8c92ad20c08fd2f48b95db7fe15bf4856b249604feedcff25 4.695657253265381 s 17 | Time elapsed for analyzing tx: 0xe99eeb28c6e4fcb0c7ee1f7adeeb03a62bf9940eb594f9a0b8e68c9b30bdf58c 3.5298120975494385 s 18 | Time elapsed for analyzing tx: 0x4a90bf0b6fc366695b347413edbca655bac5fab3ee7c756d3b61314375ea9b85 4.63828706741333 s 19 | Time elapsed for analyzing tx: 0x5c20845199bc0827287664735b222a514f52c09dc5e2efe0eaf65acc34cb9cdd 4.859069108963013 s 20 | Time elapsed for analyzing tx: 0x8102f49409ee9932abbae26257a70e1c693d5593c0c3fd4b1d286b88f2bcb60d 5.184396982192993 s 21 | Time elapsed for analyzing tx: 0x24f87b4b8514b2d2b0c7d6bb85f193613f056e02ec9711636a1d382b3832c385 3.793644905090332 s 22 | Time elapsed for analyzing tx: 0xe596cb1d6a61f5cb85bc116b30c8b72935822d292d5df55c926c02322baf7efd 4.596722841262817 s 23 | Time elapsed for analyzing tx: 0xc43d5cf6ea9f0bae9fe854d69dd019fa8ca042785f0e48bec9341a1dd64c2da5 3.7961769104003906 s 24 | Time elapsed for analyzing tx: 0x4a0944c4c19fb19dfe4f359a261b53537f1360daac8dd41f8bb6a3f4ff922635 4.602333068847656 s 25 | Time elapsed for analyzing tx: 0x2e23285e8e63581d17bba3622a4901d0156872a1836f2de3211273dea12558ba 3.7479419708251953 s 26 | Time elapsed for analyzing tx: 0xfdc8a9134dc5db7c6ab61dbd79efbcdd6c27c4e0af8c8895d541f8aab9f3c5dd 3.618295192718506 s 27 | Time elapsed for analyzing tx: 0x998fb8c147545b2f4d94e528264e346f5fe55582a7709da5429856c650cc30a9 4.538674831390381 s 28 | Time elapsed for analyzing tx: 0x526eed50ac5eb515a68917523ef7a5580168ed0fd9777c4242319a5de20d7f49 3.8345680236816406 s 29 | Time elapsed for analyzing tx: 0xd4613d5e315330be193b123095370599beeaede031bfc2d128b8bb19e40565a5 4.539416074752808 s 30 | Time elapsed for analyzing tx: 0x8fec427305c11d17fb40ee50b30f765abcde7dd91af46b6fe9cffc840a2085fb 3.7190098762512207 s 31 | Time elapsed for analyzing tx: 0x78bbb5974b677afc981500e3eaa77938fdd100defb7096810f02050b4dfe6e65 4.687687158584595 s 32 | Time elapsed for analyzing tx: 0x812f2d4afdff2779ab8d62d47355e004853056927133b8858d2e6f3ddd758dcf 4.228569984436035 s 33 | Time elapsed for analyzing tx: 0x27524945d17f1b172d3a738bb9314d559ff08c22d4e1b592bb55b1b628be8c32 4.362161159515381 s 34 | Time elapsed for analyzing tx: 0x3a9d59d11d84e9d9b226c49364c6cfe8ebbdbe3436bc03c5dc8553ab9ef59873 3.5878288745880127 s 35 | Time elapsed for analyzing tx: 0x98038d87695405a0c9d10cbdbc47eb0721b64d52720383f8e4e859127e8b60e8 4.364042043685913 s 36 | Time elapsed for analyzing tx: 0x1a0bfc77de177cb40135c7aff873cd866b9799b4c22b6715cff4d69409b2930d 4.613483190536499 s 37 | Time elapsed for analyzing tx: 0x673053a1e3dca1344040801e8c78b30b0b5f6a9897b246bc5eb348e62347be15 4.555212020874023 s 38 | Time elapsed for analyzing tx: 0x509b134247f308ef9e190cf2a9f9b902955c4cfb120c107933162e4dfa561c9e 4.0152928829193115 s 39 | now we arrive at the function call transfer, ii = 18565 40 | Time elapsed for analyzing tx: 0x597d11c05563611cb4ad4ed4c57ca53bbe3b7d3fefc37d1ef0724ad58904742b 3.6080098152160645 s 41 | now we arrive at the function call transfer, ii = 20848 42 | Time elapsed for analyzing tx: 0x7604c7dd6e9bcdba8bac277f1f8e7c1e4c6bb57afd4ddf6a16f629e8495a0281 5.241880893707275 s 43 | no access for tx: 0x8c7cc6ab0a5ed76098152927c52a7f3d3ded6f95783fbd13e05b174574bb2763 44 | Time elapsed for extracting invariant-related data: 5.250128746032715 s 45 | ===================================================== 46 | =============== Access Control ====================== 47 | ===================================================== 48 | func initialize0xcc2a9a5b has only 1 sender 49 | func initialize0xcc2a9a5b has only 1 origin 50 | func invest0xE8B5E51F has more than 5 origins 51 | func withdrawTo0xC86283C8 has only 1 sender 52 | func withdrawTo0xC86283C8 has only 1 origin 53 | ==invariant map: 54 | { 'initialize0xCC2A9A5B': { 'isOriginManager': [ '0x3aa27ab297a3a753f79c5497569ba2dacc2bc35a', 55 | '0x1d5a56402425c1099497c1ad715a6b56aaccb72b'], 56 | 'isSenderManager': [ '0x00000000b2ff98680adaf8a3e382176bbfc34c8f', 57 | '0x1695ce70da4521cb94dea036e6ebcf1e8a073ee6'], 58 | 'require(origin==sender)': False}, 59 | 'initialize0xcc2a9a5b': { 'isOriginOwner': '0xe36cc0432619247ab12f1cdd19bb3e7a24a7f47c', 60 | 'isSenderOwner': '0xe36cc0432619247ab12f1cdd19bb3e7a24a7f47c', 61 | 'require(origin==sender)': True}, 62 | 'invest0xE8B5E51F': { 'isSenderManager': [ '0x0a548513693a09135604e78b8a8fe3bb801586e6', 63 | '0x00000000b2ff98680adaf8a3e382176bbfc34c8f', 64 | '0x1695ce70da4521cb94dea036e6ebcf1e8a073ee6'], 65 | 'require(origin==sender)': False}, 66 | 'withdrawTo0xC86283C8': { 'isOriginOwner': '0xe36cc0432619247ab12f1cdd19bb3e7a24a7f47c', 67 | 'isSenderOwner': '0x0a548513693a09135604e78b8a8fe3bb801586e6', 68 | 'require(origin==sender)': False}, 69 | 'withdrawToForge0x7B9967A5': { 'isOriginManager': [ '0x3aa27ab297a3a753f79c5497569ba2dacc2bc35a', 70 | '0x1d5a56402425c1099497c1ad715a6b56aaccb72b'], 71 | 'isSenderManager': [ '0x00000000b2ff98680adaf8a3e382176bbfc34c8f', 72 | '0x1695ce70da4521cb94dea036e6ebcf1e8a073ee6'], 73 | 'require(origin==sender)': False}} 74 | Interpretation of the above invariant map: 75 | For the function initialize0xcc2a9a5b: 76 | is the invariant require(origin==sender) satisfied? True 77 | is the invariant isSenderOwner satisfied? 0xe36cc0432619247ab12f1cdd19bb3e7a24a7f47c 78 | is the invariant isOriginOwner satisfied? 0xe36cc0432619247ab12f1cdd19bb3e7a24a7f47c 79 | For the function invest0xE8B5E51F: 80 | is the invariant require(origin==sender) satisfied? False 81 | the invariant isSenderManager has parameters ['0x0a548513693a09135604e78b8a8fe3bb801586e6', '0x00000000b2ff98680adaf8a3e382176bbfc34c8f', '0x1695ce70da4521cb94dea036e6ebcf1e8a073ee6'] 82 | For the function withdrawTo0xC86283C8: 83 | is the invariant require(origin==sender) satisfied? False 84 | is the invariant isSenderOwner satisfied? 0x0a548513693a09135604e78b8a8fe3bb801586e6 85 | is the invariant isOriginOwner satisfied? 0xe36cc0432619247ab12f1cdd19bb3e7a24a7f47c 86 | For the function initialize0xCC2A9A5B: 87 | is the invariant require(origin==sender) satisfied? False 88 | the invariant isSenderManager has parameters ['0x00000000b2ff98680adaf8a3e382176bbfc34c8f', '0x1695ce70da4521cb94dea036e6ebcf1e8a073ee6'] 89 | the invariant isOriginManager has parameters ['0x3aa27ab297a3a753f79c5497569ba2dacc2bc35a', '0x1d5a56402425c1099497c1ad715a6b56aaccb72b'] 90 | For the function withdrawToForge0x7B9967A5: 91 | is the invariant require(origin==sender) satisfied? False 92 | the invariant isSenderManager has parameters ['0x00000000b2ff98680adaf8a3e382176bbfc34c8f', '0x1695ce70da4521cb94dea036e6ebcf1e8a073ee6'] 93 | the invariant isOriginManager has parameters ['0x3aa27ab297a3a753f79c5497569ba2dacc2bc35a', '0x1d5a56402425c1099497c1ad715a6b56aaccb72b'] 94 | ===================================================== 95 | =================== Time Locks ====================== 96 | ===================================================== 97 | ==invariant map: 98 | {'checkSameOriginBlock': False, 'checkSameSenderBlock': False} 99 | Interpretation of the above invariant map: 100 | is the invariant checkSameSenderBlock satisfied? False 101 | is the invariant checkSameOriginBlock satisfied? False 102 | ===================================================== 103 | =================== Gas Control ===================== 104 | ===================================================== 105 | ==invariant map: 106 | { 'initialize0xCC2A9A5B': { 'require(gasStart - gasEnd <= constant)': 43210, 107 | 'require(gasStart <= constant)': 1441473}, 108 | 'initialize0xcc2a9a5b': {}, 109 | 'invest0xE8B5E51F': { 'require(gasStart - gasEnd <= constant)': 212948, 110 | 'require(gasStart <= constant)': 1420539}, 111 | 'withdrawTo0xC86283C8': { 'require(gasStart - gasEnd <= constant)': 186329, 112 | 'require(gasStart <= constant)': 379406}, 113 | 'withdrawToForge0x7B9967A5': { 'require(gasStart - gasEnd <= constant)': 129411, 114 | 'require(gasStart <= constant)': 1273876}} 115 | Interpretation of the above invariant map: 116 | function invest0xE8B5E51F requires gasStart <= 1420539 117 | function invest0xE8B5E51F requires gasStart - gasEnd <= 212948 118 | function withdrawTo0xC86283C8 requires gasStart <= 379406 119 | function withdrawTo0xC86283C8 requires gasStart - gasEnd <= 186329 120 | function initialize0xCC2A9A5B requires gasStart <= 1441473 121 | function initialize0xCC2A9A5B requires gasStart - gasEnd <= 43210 122 | function withdrawToForge0x7B9967A5 requires gasStart <= 1273876 123 | function withdrawToForge0x7B9967A5 requires gasStart - gasEnd <= 129411 124 | ===================================================== 125 | =================== Re-entrancy ===================== 126 | ===================================================== 127 | ==invariant list: 128 | {'NonReentrantLocks': True} 129 | Interpretation of the above invariant map: 130 | re-entrancy guard can be applied 131 | ===================================================== 132 | ================ Special Storage ==================== 133 | ===================================================== 134 | ==invariant map: 135 | {} 136 | ===================================================== 137 | =================== DataFlow ======================== 138 | ===================================================== 139 | ==invariant map: 140 | { 'callvalue': {}, 141 | 'dataFlow': { 'withdrawTo_withdraw': { 1972: [ { 'Selector': '0x70a08231', 142 | 'addr': '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', 143 | 'gas': '0x3370a', 144 | 'inputTypes': ['address'], 145 | 'inputs': ['ADDRESS'], 146 | 'name': 'balanceOf', 147 | 'outputTypes': ['uint256'], 148 | 'outputs': [9900], 149 | 'pc': 1972, 150 | 'structLogsEnd': 14701, 151 | 'structLogsStart': 14460}, 152 | (90, 9900)]}}, 153 | 'mapping': {}} 154 | Interpretation of the above invariant map: 155 | For the invariant mapping: 156 | For the invariant callvalue: 157 | For the invariant dataFlow: 158 | It can be applied to function withdrawTo_withdraw: 159 | For Data Source read from pc = 1972 160 | with lowerbound = 90 161 | with upperbound = 9900 162 | ===================================================== 163 | =================== MoneyFlow ======================= 164 | ===================================================== 165 | ==invariant map: 166 | {'withdrawTo+withdraw': {'transferAmount': (90, 9900)}} 167 | Interpretation of the above invariant map: 168 | For the invariant moneyFlow: 169 | It can be applied to function withdrawTo+withdraw: 170 | with lowerbound of transferAmount = 90 171 | with upperbound of transferAmount = 9900 172 | Time elapsed for inferring invariants: 17.50284504890442 s 173 | Time elapsed for translating logs: 6.106295108795166 s 174 | -------------------------------------------------------------------------------- /staticAnalyzer/temp2.txt: -------------------------------------------------------------------------------- 1 | 1.971790075302124 s 2 | 1.1898088455200195 s 3 | 1.124871015548706 s 4 | 3.6804418563842773 s 5 | 4.585702180862427 s 6 | 3.391124963760376 s 7 | 4.695657253265381 s 8 | 3.5298120975494385 s 9 | 4.63828706741333 s 10 | 4.859069108963013 s 11 | 5.184396982192993 s 12 | 3.793644905090332 s 13 | 4.596722841262817 s 14 | 3.7961769104003906 s 15 | 4.602333068847656 s 16 | 3.7479419708251953 s 17 | 3.618295192718506 s 18 | 4.538674831390381 s 19 | 3.8345680236816406 s 20 | 4.539416074752808 s 21 | 3.7190098762512207 s 22 | 4.687687158584595 s 23 | 4.228569984436035 s 24 | 4.362161159515381 s 25 | 3.5878288745880127 s 26 | 4.364042043685913 s 27 | 4.613483190536499 s 28 | 4.555212020874023 s 29 | 4.0152928829193115 s 30 | 3.6080098152160645 s 31 | 5.241880893707275 s -------------------------------------------------------------------------------- /staticAnalyzer/vyperAnalyzer.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 4 | sys.path.append(os.path.dirname(SCRIPT_DIR)) 5 | 6 | 7 | from crawlPackage.crawlEtherscan import CrawlEtherscan 8 | # # from crawlPackage.crawlQuicknode import CrawlQuickNode 9 | # # from crawlPackage.crawlTrueBlocks import CrawlTrueBlocks 10 | import subprocess, json, time, random, pickle, math, copy, toml 11 | 12 | import pprint 13 | 14 | from web3 import Web3, HTTPProvider 15 | from packaging.version import Version 16 | from os import listdir 17 | from os.path import isfile, join 18 | 19 | 20 | from slither.core.solidity_types.elementary_type import ElementaryType 21 | from slither.core.solidity_types.user_defined_type import UserDefinedType 22 | from slither.core.solidity_types.mapping_type import MappingType 23 | from slither.core.solidity_types.array_type import ArrayType 24 | from slither.core.solidity_types.function_type import FunctionType 25 | 26 | 27 | from eth_abi import decode 28 | import sqlite3 29 | from crawlPackage.cacheDatabase import _save_transaction_receipt, _load_transaction_receipt, _save_contract, _load_contract 30 | 31 | 32 | 33 | # def writeStats(contractAddress, ContractName, CompilerVersion: str, EVMVersion: str, sourceCode: str): 34 | # contractAddress = contractAddress.lower() 35 | # path = SCRIPT_DIR + "/cache/" + contractAddress 36 | # # print(path) 37 | # if not os.path.exists(path): 38 | # os.makedirs(path) 39 | 40 | # with open(path + "/" + ContractName + ".vy", "w") as f: 41 | # content = "# " + CompilerVersion + "\n" 42 | # content += "# " + EVMVersion + "\n\n" 43 | # # replace @version with @version_ to avoid compiler complaining 44 | # content += sourceCode.replace("@version", "version") 45 | # f.write(content) 46 | 47 | 48 | def readStats(contractAddress, cur): 49 | contractAddress = contractAddress.lower() 50 | # path = SCRIPT_DIR + "/cache/" + contractAddress + "/" 51 | # if not os.path.exists(path): 52 | # return None, None 53 | 54 | contract_dict = _load_contract(contractAddress, cur) 55 | if contract_dict is None: 56 | return None, None 57 | else: 58 | return contract_dict["CompilerVersion"], contract_dict["EVMVersion"] 59 | 60 | 61 | 62 | class vyperAnalyzer: 63 | 64 | def __init__(self) -> None: 65 | self.etherScan = CrawlEtherscan() 66 | self.cur = None 67 | 68 | 69 | def contract2Sourcecode(self, contractAddress: str): 70 | """Given a contract address, return the source code""" 71 | # check if the source code is already in the cache 72 | CompilerVersion, EVMVersion = readStats(contractAddress, self.cur) 73 | if CompilerVersion is not None and EVMVersion is not None: 74 | return CompilerVersion, EVMVersion 75 | results = self.etherScan.Contract2Sourcecode(contractAddress) 76 | if "vyper" not in results["CompilerVersion"]: 77 | return None, None 78 | if len(results) > 1: 79 | sys.exit("vyperAnalyzer: cannot handle multiple Vyper contracts in one contract address") 80 | 81 | result = results[0] 82 | sourceCode = result["SourceCode"] 83 | ABI = result["ABI"] 84 | ContractName = result["ContractName"] 85 | CompilerVersion = result["CompilerVersion"] 86 | EVMVersion = result["EVMVersion"] 87 | 88 | # writeStats(contractAddress, ContractName, CompilerVersion, EVMVersion, sourceCode) 89 | 90 | return CompilerVersion, EVMVersion 91 | 92 | 93 | def handleSourceAndVersion(self, contractAddress: str) -> list: 94 | CompilerVersion, EVMVersion = self.contract2Sourcecode(contractAddress) 95 | 96 | if EVMVersion != "Default": 97 | sys.exit("vyperAnalyzer: cannot handle non-default EVM version for now") 98 | 99 | compilerVersion = Version(CompilerVersion) 100 | storageLayoutVersion = Version("0.2.16") 101 | 102 | useCompiler = CompilerVersion 103 | if compilerVersion < storageLayoutVersion: 104 | useCompiler = "0.2.16" # Vyper starts to support storage layout option from 0.2.16 105 | 106 | subprocess.run(["vyper-select", "use", useCompiler], stdout=subprocess.PIPE, check=True) 107 | 108 | 109 | 110 | def Contract2funcSigMap(self, contractAddress: str) -> list: 111 | """Given a contract address, return a list of funcSigMap""" 112 | self.handleSourceAndVersion(contractAddress) 113 | 114 | mypath = os.path.join(SCRIPT_DIR, "cache", contractAddress.lower()) 115 | onlyfiles = [f for f in listdir(mypath) if isfile(join(mypath, f))] 116 | onlyfiles = [f for f in onlyfiles if f.endswith(".vy")] 117 | if len(onlyfiles) != 1: 118 | sys.exit("vyperAnalyzer: Error: multiple files in the same folder {}!".format(mypath)) 119 | 120 | test_item = os.path.join(mypath, onlyfiles[0]) 121 | abi = None 122 | with subprocess.Popen(["vyper", test_item, "-f", "abi"], stdout=subprocess.PIPE) as process: 123 | counter = 0 124 | for line in process.stdout: 125 | counter += 1 126 | if counter > 2: 127 | sys.exit("vyperAnalyzer: Error: vyper compile abi exceeds 2 lines!") 128 | if counter == 1: 129 | abi = json.loads(line) 130 | 131 | functionSigMap = self.etherScan.Contract2funcSigMap(contractAddress, abi) 132 | return functionSigMap 133 | 134 | 135 | 136 | def Contract2storageLayout(self, contractAddress: str) -> list: 137 | """Given a contract address, return a list of storageLayout""" 138 | self.handleSourceAndVersion(contractAddress) 139 | 140 | mypath = os.path.join(SCRIPT_DIR, "cache", contractAddress.lower()) 141 | onlyfiles = [f for f in listdir(mypath) if isfile(join(mypath, f))] 142 | onlyfiles = [f for f in onlyfiles if f.endswith(".vy")] 143 | if len(onlyfiles) != 1: 144 | sys.exit("vyperAnalyzer: Error: multiple files in the same folder {}!".format(mypath)) 145 | 146 | test_item = os.path.join(mypath, onlyfiles[0]) 147 | storageLayout = None 148 | with subprocess.Popen(["vyper", test_item, "-f", "layout"], stdout=subprocess.PIPE) as process: 149 | counter = 0 150 | for line in process.stdout: 151 | counter += 1 152 | if counter > 2: 153 | sys.exit("vyperAnalyzer: Error: vyper compile layout exceeds 2 lines!") 154 | if counter == 1: 155 | storageLayout = json.loads(line) 156 | # print(storageLayout) 157 | return storageLayout 158 | 159 | 160 | 161 | def parseType(self, typeStr): 162 | # eg. HashMap[address, HashMap[address, uint256][address, uint256]][address, HashMap[address, uint256][address, uint256]] 163 | # Find the position of ] which matches the first [ 164 | # Check if typeStr is a basic type 165 | # remove all whitespaces 166 | typeStr = typeStr.replace(" ", "") 167 | 168 | 169 | try: 170 | ElementaryType(typeStr) 171 | return typeStr 172 | except: 173 | pass 174 | 175 | # Mapping 176 | if typeStr[:7] == "HashMap": 177 | # print(typeStr) 178 | pos = 0 179 | commaPos = 0 180 | for i in range(len(typeStr)): 181 | if typeStr[i] == "[": 182 | pos += 1 183 | elif typeStr[i] == "]": 184 | pos -= 1 185 | if pos == 0: 186 | break 187 | elif typeStr[i] == ",": 188 | if pos == 1: 189 | commaPos = i 190 | keyType = typeStr[8:commaPos] 191 | valueType = typeStr[commaPos+1:i] 192 | _type_stored = (self.parseType(keyType), self.parseType(valueType)) 193 | return _type_stored 194 | 195 | else: 196 | if typeStr[-1] == "]" and typeStr[-2] != "[": 197 | # Fixed Size Array 198 | leftBracketPos = -1 199 | rightBracketPos = -1 200 | for i in range(len(typeStr)): 201 | if typeStr[i] == "[": 202 | if leftBracketPos == -1: 203 | leftBracketPos = i 204 | else: 205 | sys.exit("vyperAnalyzer: Error: cannot handle nested array!, typeStr: {}".format(typeStr)) 206 | elif typeStr[i] == "]": 207 | if rightBracketPos == -1: 208 | rightBracketPos = i 209 | else: 210 | sys.exit("vyperAnalyzer: Error: cannot handle nested array!, typeStr: {}".format(typeStr)) 211 | 212 | elementType = typeStr[:leftBracketPos] 213 | arraySize = int(typeStr[leftBracketPos+1:rightBracketPos]) 214 | _type_stored = self.parseType(elementType) + "[" + str(arraySize) + "]" 215 | return _type_stored 216 | elif typeStr[-1] == "]" and typeStr[-2] == "[": 217 | # Dynamic Array 218 | return self.parseType(typeStr[:-2]) + "[]" 219 | else: 220 | if typeStr == "String": 221 | # Not sure if it;s correct but keep it for now 222 | return "string" 223 | 224 | elif typeStr == "VotedSlope" or typeStr == "Point": 225 | # Hope it is not used in later processing 226 | return typeStr 227 | 228 | elif typeStr == "nonreentrantlock" or typeStr == "LockedBalance": 229 | return typeStr 230 | 231 | else: 232 | # Hope it is not used in later processing 233 | return typeStr 234 | 235 | # else: 236 | # sys.exit("vyperAnalyzer: Error: cannot handle type {}, typeStr: {}".format(typeStr, typeStr)) 237 | 238 | 239 | 240 | def StorageLayout2StorageMapping(self, storageLayout: list, is024: bool = False) -> list: 241 | """Given a storageLayout, return a list of storageMapping""" 242 | """is024: if Vyper version is 0.2.4""" 243 | # eg. {'name': {'type': 'String[64]', 'location': 'storage', 'slot': 0}, 244 | # 'symbol': {'type': 'String[32]', 'location': 'storage', 'slot': 4}, 245 | # 'decimals': {'type': 'uint256', 'location': 'storage', 'slot': 7}, 246 | # 'balanceOf': {'type': 'HashMap[address, uint256][address, uint256]', 'location': 'storage', 'slot': 8}, 247 | # 'allowances': {'type': 'HashMap[address, HashMap[address, uint256][address, uint256]][address, HashMap[address, uint256][address, uint256]]', 248 | # 'location': 'storage', 'slot': 9}, 249 | # 'total_supply': {'type': 'uint256', 'location': 'storage', 'slot': 10}, 250 | # 'minter': {'type': 'address', 'location': 'storage', 'slot': 11}} 251 | storageMapping = {} 252 | if is024: 253 | # sort storageLayout by its value["slot"] 254 | storageLayout = sorted(storageLayout.items(), key=lambda x: x[1]["slot"]) 255 | counter = 0 256 | for item in storageLayout: 257 | varName = item[0] 258 | value = item[1] 259 | if value["location"] == "storage": 260 | _type_stored = self.parseType(value["type"]) 261 | storageMapping[counter * 32] = (varName, _type_stored) 262 | counter += 1 263 | else: 264 | sys.exit("vyperAnalyzer: Error: location is not storage!") 265 | return storageMapping 266 | 267 | for varName, value in storageLayout.items(): 268 | if value["location"] == "storage": 269 | _type_stored = self.parseType(value["type"]) 270 | storageMapping[value["slot"] * 32] = (varName, _type_stored) # 32 is the size of a slot 271 | else: 272 | sys.exit("vyperAnalyzer: Error: cannot handle non-storage type [{}] yet!".format(value["location"])) 273 | return storageMapping 274 | 275 | 276 | def Contract2storageMapping(self, contractAddress: str) -> dict: 277 | """Given a contract address, return a dict of storageMapping""" 278 | CompilerVersion, _ = self.contract2Sourcecode(contractAddress) 279 | storageLayout = self.Contract2storageLayout(contractAddress) 280 | if storageLayout is None: 281 | return None 282 | is024 = (CompilerVersion == "0.2.4") 283 | storageMapping = self.StorageLayout2StorageMapping(storageLayout, is024=is024) 284 | return storageMapping 285 | 286 | 287 | 288 | 289 | 290 | 291 | if __name__ == "__main__": 292 | 293 | analyzer = vyperAnalyzer() 294 | Curve3PoolContractAddress = "0xbEbc44782C7dB0a1A60Cb6fe97d0b483032FF1C7" 295 | 296 | 297 | CompilerVersion, EVMVersion = analyzer.contract2Sourcecode(Curve3PoolContractAddress) 298 | # print(sourceCode) 299 | 300 | # functionSigMap = analyzer.Contract2funcSigMap(Curve3PoolContractAddress) 301 | # print(functionSigMap) 302 | print(CompilerVersion) 303 | print(EVMVersion) 304 | 305 | # funcSigMap = analyzer.Contract2funcSigMap(Curve3PoolContractAddress) 306 | # print(funcSigMap) 307 | 308 | # # EMN contract 309 | # contractAddress = "0x5ade7aE8660293F2ebfcEfaba91d141d72d221e8" 310 | 311 | # # test Contract2storageLayout 312 | # storageLayout = analyzer.Contract2storageLayout("0xbfcf63294ad7105dea65aa58f8ae5be2d9d0952a") 313 | # print(storageLayout) 314 | # storageMapping = analyzer.Contract2storageMapping("0xbfcf63294ad7105dea65aa58f8ae5be2d9d0952a") 315 | # print(storageMapping) 316 | 317 | # # store storageMapping into a json file 318 | # # pretty print 319 | 320 | # with open("storageMapping.json", "w") as f: 321 | # json.dump(storageMapping, f, indent=4) 322 | 323 | # # # test parseType 324 | # # type1 = "String[64]" 325 | # # type2 = "String[32]" 326 | # # type3 = "uint256" 327 | # # type4 = "HashMap[address, uint256][address, uint256]" 328 | # # type5 = "HashMap[address, HashMap[address, uint256][address, uint256]][address, HashMap[address, uint256][address, uint256]]" 329 | # # type6 = "address" 330 | # # types = [type1, type2, type3, type4, type5, type6] 331 | # # for typeStr in types: 332 | # # print(analyzer.parseType(typeStr)) 333 | 334 | # # test StorageLayout2StorageMapping 335 | 336 | -------------------------------------------------------------------------------- /trackerPackage/dataSource.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 4 | sys.path.append(os.path.dirname(SCRIPT_DIR)) 5 | import copy 6 | 7 | 8 | # we have to keep an record of which bytes correspond to dataSource 9 | class dataSource: 10 | """It represents some end points data source""" 11 | def __init__(self, d: dict = None, opcode = None): 12 | self.sources = [] 13 | self.children = [] 14 | self.metaData = {} 15 | if d is not None: 16 | self.sources.append(d) 17 | self.children.append(None) 18 | if opcode is not None: 19 | self.sources.append(opcode) 20 | self.children.append(None) 21 | 22 | def find(self, name: str): 23 | for source in self.sources: 24 | if isinstance(source, dict): 25 | if "name" in source.keys() and source["name"] == name: 26 | return True 27 | elif isinstance(source, str): 28 | if source == name: 29 | return True 30 | elif isinstance(source, tuple): 31 | for jj in range(len(source)): 32 | if source[jj] == name: 33 | return True 34 | return 35 | 36 | def remove(self, name: str): 37 | for ii in range(len(self.sources)): 38 | if not isinstance(self.sources[ii], str): 39 | continue 40 | elif (name == "CALLER" or name == "ADDRESS" or name == "ORIGIN") and self.sources[ii].startswith(name): 41 | self.sources.pop(ii) 42 | self.children.pop(ii) 43 | return 44 | elif self.sources[ii] == name: 45 | self.sources.pop(ii) 46 | self.children.pop(ii) 47 | return 48 | 49 | 50 | def to_dict(self): 51 | children = [child.to_dict() if child is not None else None for child in self.children] 52 | return {"sources": self.sources, "children": children, "metaData": self.metaData} 53 | 54 | def addChild(self, child): 55 | self.children.append(child) 56 | 57 | def isEmpty(self): 58 | return len(self.sources) == 0 59 | 60 | def addFunc(self, d: dict, child = None): 61 | isEqual = False 62 | for item in self.sources: 63 | if item == d: 64 | isEqual = True 65 | break 66 | if not isEqual: 67 | self.sources.append(d) 68 | self.children.append(child) 69 | 70 | def addOpcode(self, opcode: str, child = None): 71 | if opcode not in self.sources: 72 | self.sources.append(opcode) 73 | self.children.append(child) 74 | 75 | def merge(self, other): 76 | for ii in range(len(other.sources)): 77 | source = other.sources[ii] 78 | child = other.children[ii] 79 | if isinstance(source, dict): 80 | self.addFunc(source, child) 81 | else: 82 | self.addOpcode(source, child) 83 | 84 | def endPoints(self) -> list: 85 | endPoints = [] 86 | for ii in range(len(self.sources)): 87 | source = self.sources[ii] 88 | if self.children[ii] == None: 89 | endPoints.append(source) 90 | else: 91 | endPoints += self.children[ii].endPoints() 92 | return endPoints 93 | 94 | def __str__(self) -> str: 95 | string = "[" 96 | endPoints = self.endPoints() 97 | for ii in range(len(endPoints)): 98 | if ii != 0: 99 | string += ", " 100 | if isinstance(endPoints[ii], tuple): 101 | if endPoints[ii][0] == "msg.data": 102 | string += "msg.data+" + str(endPoints[ii][1]) + "-" + str(endPoints[ii][2]) 103 | elif endPoints[ii][1] == "SLOAD": 104 | key = endPoints[ii][2] 105 | if endPoints[ii][2][0:2] == "0x": 106 | string += "SLOAD+" + str(endPoints[ii][2][-8:]) 107 | else: 108 | string += "SLOAD+" + str(endPoints[ii][2]) 109 | elif endPoints[ii][1] == "Mapping": 110 | string += "SLOAD+(Mapping" + str(endPoints[ii][2][-4:]) + "[" + str(endPoints[ii][3]) + "]" + ('+' if endPoints[ii][4] >= 0 else '') + str(endPoints[ii][4]) + ")" 111 | elif endPoints[ii][0] == "PC" or endPoints[ii][0] == "TIMESTAMP" or endPoints[ii][0] == "CALLER" or endPoints[ii][0] == "ORIGIN" or \ 112 | endPoints[ii][0] == "RETURNDATASIZE" or endPoints[ii][0] == "ADDRESS" or endPoints[ii][0] == "GAS" or endPoints[ii][0] == "CALLDATASIZE" or \ 113 | endPoints[ii][0] == "SELFBALANCE" or endPoints[ii][0] == "CALLVALUE" or endPoints[ii][0] == "NUMBER" or endPoints[ii][0] == "BALANCE": 114 | string += endPoints[ii][0] 115 | elif endPoints[ii][0] == "SHA3-64": 116 | string += endPoints[ii][0] + "-" + endPoints[ii][1][-4:] 117 | elif endPoints[ii][0] == "SHA3": 118 | string += endPoints[ii][0] 119 | elif endPoints[ii][0] == "address(this).code": 120 | string += endPoints[ii][0] + "[{}:{}]".format(endPoints[ii][1], endPoints[ii][2]) 121 | else: 122 | print(endPoints[ii]) 123 | sys.exit("dataSource: Error: unknown tuple type") 124 | elif isinstance(endPoints[ii], dict): 125 | string += str(endPoints[ii]["name"]) + "(" + str(endPoints[ii]["inputs"]) + ")" 126 | 127 | else: 128 | string += str(endPoints[ii]) 129 | string += "]" 130 | return string 131 | 132 | 133 | 134 | 135 | 136 | def getEndPoints(dataS: dict): 137 | endPoints = [] 138 | for ii in range(len(dataS["sources"])): 139 | source = dataS["sources"][ii] 140 | if dataS["children"][ii] == None: 141 | endPoints.append(source) 142 | else: 143 | endPoints += getEndPoints(dataS["children"][ii]) 144 | return endPoints -------------------------------------------------------------------------------- /trackerPackage/memoryTracker.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 4 | sys.path.append(os.path.dirname(SCRIPT_DIR)) 5 | from trackerPackage.dataSource import * 6 | from trackerPackage.stackTracker import * 7 | 8 | class memoryTracker: 9 | """It represents some memory locations where some data source is stored""" 10 | def __init__(self): 11 | self.memoryMap = [] 12 | 13 | def find(self, name: str): 14 | for (start, end, dataSrcInfo) in self.memoryMap: 15 | if dataSrcInfo.find(name): 16 | return True 17 | return False 18 | 19 | def overwriteStackEntry(self, start, end, entry: stackEntry): 20 | self.overwriteInterval(start, end, dataSource()) 21 | if end - start != entry.length: 22 | entry.shiftInterval( -1 * (entry.length - (end - start)) ) 23 | # sys.exit("memoryTracker Error: stack entry length does not match {} and {}".format(end - start, entry.length)) 24 | for dataSrcInfo in entry.dataSrcMap: 25 | self.addInterval(start + dataSrcInfo[0], start + dataSrcInfo[1], dataSrcInfo[2]) 26 | 27 | def addInterval(self, start, end, dataSrcInfo: dataSource): 28 | self.memoryMap.append([start, end, dataSrcInfo]) 29 | 30 | def overwriteInterval(self, start, end, dataSrcInfo: dataSource): 31 | # if start < 324 + 100 and end > 324: 32 | # print("now is the time") 33 | 34 | # interval is of shape (start, end, {data source info}) 35 | hasIt = False 36 | for ii in range(len(self.memoryMap)): 37 | item = self.memoryMap[ii] 38 | # no overlapping 39 | if item[1] <= start: 40 | continue 41 | elif item[0] >= end: 42 | continue 43 | # overlapping 44 | if start <= item[0] and end >= item[1]: 45 | # new interval is larger than the existing interval 46 | # remove the existing interval 47 | if not hasIt: 48 | item[0] = start 49 | item[1] = end 50 | item[2] = dataSrcInfo 51 | hasIt = True 52 | else: 53 | item[0] = None 54 | elif start <= item[0] and end < item[1]: 55 | item[0] = end 56 | elif start > item[0] and end >= item[1]: 57 | item[1] = start 58 | elif start > item[0] and end < item[1]: 59 | # split the existing interval 60 | self.memoryMap.append([end, item[1], item[2]]) 61 | item[1] = start 62 | if not hasIt: 63 | self.memoryMap.append([start, end, dataSrcInfo]) 64 | 65 | # remove all items whose start is None 66 | self.memoryMap = [item for item in self.memoryMap if item[0] is not None] 67 | 68 | def getInterval(self, start, end): 69 | dataSrcInfo = dataSource() 70 | for item in self.memoryMap: 71 | # check if overlapping 72 | if item[1] <= start: 73 | continue 74 | elif item[0] >= end: 75 | continue 76 | else: 77 | # overlapping 78 | dataSrcInfo.merge(item[2]) 79 | return dataSrcInfo 80 | 81 | def getIntervalDetails(self, start, end): 82 | dataSrcVec = [] # a list of data sources 83 | for item in self.memoryMap: 84 | # check if overlapping 85 | if item[1] <= start: 86 | continue 87 | elif item[0] >= end: 88 | continue 89 | else: 90 | # overlapping 91 | dataSrcVec.append( (item[0] - start, item[1] - start, item[2]) ) 92 | return dataSrcVec 93 | 94 | 95 | 96 | def __str__(self) -> str: 97 | return str(self.memoryMap) 98 | 99 | -------------------------------------------------------------------------------- /trackerPackage/stackTracker.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 4 | sys.path.append(os.path.dirname(SCRIPT_DIR)) 5 | from trackerPackage.dataSource import * 6 | import copy 7 | from itertools import groupby 8 | 9 | 10 | class stackEntry: 11 | def __init__(self, length = 0, dataSrcOrdataSrcVec = None) -> None: 12 | self.dataSrcMap = [] 13 | self.length = length # data length in bytes 14 | if dataSrcOrdataSrcVec is None: 15 | return 16 | elif isinstance(dataSrcOrdataSrcVec, dataSource): 17 | self.dataSrcMap = [[0, length, dataSrcOrdataSrcVec]] 18 | elif isinstance(dataSrcOrdataSrcVec, list): 19 | self.dataSrcMap = copy.deepcopy(dataSrcOrdataSrcVec) 20 | else: 21 | sys.exit("stackEntry Error: dataSrcOrdataSrcVec is neither a dataSource nor a list") 22 | 23 | def gc(self): 24 | sources = [] 25 | for ii in reversed(range(len(self.dataSrcMap))): 26 | if self.dataSrcMap[ii][2].isEmpty(): 27 | del self.dataSrcMap[ii] 28 | 29 | 30 | 31 | 32 | def mergeList(self, dataSrcVec: list): 33 | for item in dataSrcVec: 34 | self.addInterval(item[0], item[1], item[2]) 35 | self.gc() 36 | 37 | def merge(self, other): 38 | # merge other into self 39 | for item in other.dataSrcMap: 40 | self.addInterval(item[0], item[1], item[2]) 41 | self.gc() 42 | 43 | def addInterval(self, start, end, dataSrcInfo: dataSource): 44 | for i_start, i_end, i_dataSrcInfo in self.dataSrcMap: 45 | if i_start == start and i_end == end: 46 | i_dataSrcInfo.merge(dataSrcInfo) 47 | return 48 | self.dataSrcMap.append((start, end, dataSrcInfo)) 49 | self.gc() 50 | 51 | 52 | def overwriteInterval(self, start, end, dataSrcInfo: dataSource): 53 | hasIt = False 54 | for ii in range(len(self.dataSrcMap)): 55 | item = self.dataSrcMap[ii] 56 | # no overlapping 57 | if item[1] <= start: 58 | continue 59 | elif item[0] >= end: 60 | continue 61 | # overlapping 62 | if start <= item[0] and end >= item[1]: 63 | # new interval is larger than the existing interval 64 | # remove the existing interval 65 | if not hasIt: 66 | self.dataSrcMap[ii] = (start, end, dataSrcInfo) 67 | hasIt = True 68 | else: 69 | self.dataSrcMap[ii] = (None, item[1], item[2]) 70 | elif start <= item[0] and end < item[1]: 71 | self.dataSrcMap[ii] = (end, item[1], item[2]) 72 | elif start > item[0] and end >= item[1]: 73 | self.dataSrcMap[ii] = (item[0], start, item[2]) 74 | elif start > item[0] and end < item[1]: 75 | # split the existing interval 76 | self.dataSrcMap.append([end, item[1], item[2]]) 77 | self.dataSrcMap[ii] = (item[0], start, item[2]) 78 | if not hasIt: 79 | self.dataSrcMap.append((start, end, dataSrcInfo)) 80 | 81 | # remove all items whose start is None 82 | self.dataSrcMap = [item for item in self.dataSrcMap if item[0] is not None] 83 | self.gc() 84 | 85 | 86 | def getInterval(self, start = -1, end = 99999999): 87 | dataSrcInfo = dataSource() 88 | for item in self.dataSrcMap: 89 | # check if overlapping 90 | if item[1] <= start: 91 | continue 92 | elif item[0] >= end: 93 | continue 94 | else: 95 | # overlapping 96 | dataSrcInfo.merge(item[2]) 97 | return dataSrcInfo 98 | 99 | def removeInterval(self, start, end): 100 | self.overwriteInterval(start, end, dataSource()) 101 | self.gc() 102 | 103 | 104 | def shiftInterval(self, shift): 105 | # shift <0 means shift to the left 106 | # shift >0 means shift to the right 107 | index2remove = [] 108 | for ii in range(len(self.dataSrcMap)): 109 | item = self.dataSrcMap[ii] 110 | new_start = min(max(0, item[0] + shift), 32) 111 | new_end = min(max(0, item[1] + shift), 32) 112 | if new_start == 0 and new_end == 0: 113 | index2remove.append(ii) 114 | elif new_start == 32 and new_end == 32: 115 | index2remove.append(ii) 116 | else: 117 | self.dataSrcMap[ii] = (new_start, new_end, item[2]) 118 | 119 | # remove all items with index in index2remove from self.dataSrcMap 120 | for ii in reversed(index2remove): 121 | del self.dataSrcMap[ii] 122 | self.gc() 123 | 124 | 125 | def __str__(self) -> str: 126 | string = "[" 127 | for ii, item in enumerate(self.dataSrcMap): 128 | if ii != 0: 129 | string += ", " 130 | string += "[{}, {}, {}]".format(item[0], item[1], str(item[2]) ) 131 | string += "]" 132 | self.dataSrcMap = [list(x) for x in {(tuple(e)) for e in self.dataSrcMap }] 133 | return string 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | class stackTracker: 143 | def __init__(self, oldStack = []) -> None: 144 | self.stack = oldStack.copy() 145 | 146 | def find(self, dataSrcInfo: dataSource): 147 | # find the data source in the stack 148 | for i in range(len(self.stack)): 149 | for (start, end, item) in self.stack[i].dataSrcMap: 150 | if item == dataSrcInfo: 151 | return i 152 | return -1 153 | 154 | def __str__(self) -> str: 155 | return str(self.stack) 156 | 157 | def swap(self, n: int): 158 | # swap the last element with the element at index - n - 1 159 | if not len(self.stack) >= n + 1: 160 | sys.exit("Tracker Error: swap() called with n = {} but stack length is {}".format(n, len(self.stack))) 161 | if n == 0: 162 | pass 163 | else: 164 | self.stack[-1], self.stack[-n-1] = self.stack[-n-1], self.stack[-1] 165 | 166 | def dup(self, n: int): 167 | # n = 1: duplicate the last element 168 | # n = 2: duplicate the second last element 169 | # n = 3: duplicate the third last element 170 | if not len(self.stack) >= 1: 171 | sys.exit("Tracker Error: dup() called with n = {} but stack length is {}".format(n, len(self.stack))) 172 | if n == 0: 173 | pass 174 | else: 175 | self.stack.append( copy.deepcopy(self.stack[-n]) ) 176 | 177 | def push(self, n: stackEntry): 178 | # push n to the stack 179 | # TODO 180 | self.stack += [n] 181 | 182 | def pop(self, n: int): 183 | # pop the last n elements 184 | if not len(self.stack) >= n: 185 | sys.exit("Tracker Error: pop() called with n = {} but stack length is {}".format(n, len(self.stack))) 186 | if n == 0: 187 | pass 188 | elif n == 1: 189 | last = self.stack[-1] 190 | self.stack = self.stack[:-1] 191 | if self.stack: 192 | return last 193 | else: 194 | self.stack = self.stack[:-n] 195 | 196 | 197 | def merge_last_n(self, n: int, length: int): 198 | # Check if the stack is empty or if n is 0 or negative 199 | if not self.stack or n <= 0: 200 | return self.stack 201 | 202 | if not len(self.stack) >= n: 203 | sys.exit("Tracker Error: merge_last_n() called with n = {} but stack length is {}".format(n, len(self.stack))) 204 | # Create a new stack to store the result 205 | result_stack = [] 206 | 207 | # Loop through the elements in the stack, skipping the last n elements 208 | for i in range(len(self.stack) - n): 209 | result_stack.append( copy.deepcopy(self.stack[i]) ) 210 | 211 | # Merge the last n elements in the original stack 212 | merged_entry = stackEntry() 213 | merged_entry.length = length 214 | for i in range(len(self.stack) - n, len(self.stack)): 215 | merged_entry.merge(self.stack[i]) 216 | if isinstance(self.stack[i], set): 217 | sys.exit("Tracker Error: merge_last_n() called with set") 218 | elif isinstance(self.stack[i], dict): 219 | sys.exit("Tracker Error: merge_last_n() called with dict") 220 | 221 | 222 | result_stack.append(merged_entry) 223 | 224 | self.stack = result_stack 225 | 226 | -------------------------------------------------------------------------------- /trackerPackage/storageTracker.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 4 | sys.path.append(os.path.dirname(SCRIPT_DIR)) 5 | from trackerPackage.dataSource import * 6 | 7 | class storageTracker(): 8 | """It represents some storage locations where some data source is stored""" 9 | # sstore only writes a single word, 32 bytes 10 | # sload only reads a single word, 32 bytes 11 | def __init__(self): 12 | self.storageMap = [] 13 | 14 | def store(self, start, dataSrcInfo): 15 | # store the data source info at the start location 16 | self.storageMap.append([start, start + 32, dataSrcInfo]) 17 | 18 | def read(self, start): 19 | # read the data source info at the start location 20 | dataSrcInfo = dataSource() 21 | for item in self.storageMap: 22 | if item[1] <= start: 23 | continue 24 | elif item[0] >= start + 32: 25 | continue 26 | else: 27 | dataSrcInfo.merge(item[2]) 28 | return dataSrcInfo 29 | 30 | def readDetails(self, start): 31 | dataSrcVec = [] 32 | for item in self.storageMap: 33 | if item[1] <= start: 34 | continue 35 | elif item[0] >= start + 32: 36 | continue 37 | else: 38 | dataSrcVec.append( (item[0] - start, item[1] - start, item[2]) ) 39 | return dataSrcVec 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /utilsPackage/compressor.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import gzip 4 | import time 5 | import json 6 | import pickle 7 | 8 | 9 | def writeDataSource(SCRIPT_DIR, contract, tx, dataSourceMapList): 10 | path = SCRIPT_DIR + "/../cache/" + contract + "/{}.pickle".format(tx) 11 | # print("write", path) 12 | # print(dataSourceMapList) 13 | with open(path, 'wb') as f: 14 | pickle.dump(dataSourceMapList, f) 15 | 16 | 17 | def readDataSource(SCRIPT_DIR, contract = None, tx = None): 18 | path = None 19 | if contract is None and tx is None: 20 | path = SCRIPT_DIR 21 | else: 22 | path = SCRIPT_DIR + "/../cache/" + contract + "/{}.pickle".format(tx) 23 | objects = [] 24 | # check if file exists 25 | if not os.path.exists(path): 26 | return objects 27 | with (open(path, "rb")) as openfile: 28 | while True: 29 | try: 30 | objects.append(pickle.load(openfile)) 31 | except EOFError: 32 | break 33 | return objects 34 | 35 | 36 | def writeAccessList(SCRIPT_DIR, contract, tx, accessList): 37 | path = SCRIPT_DIR + "/../cache/" + contract + "_Access/{}.pickle".format(tx) 38 | # print("write", path) 39 | # print(dataSourceMapList) 40 | with open(path, 'wb') as f: 41 | pickle.dump(accessList, f) 42 | 43 | def readAccessList(SCRIPT_DIR, contract, tx): 44 | path = SCRIPT_DIR + "/../cache/" + contract + "_Access/{}.pickle".format(tx) 45 | objects = [] 46 | # check if file exists 47 | if not os.path.exists(path): 48 | return objects 49 | with (open(path, "rb")) as openfile: 50 | while True: 51 | try: 52 | objects.append(pickle.load(openfile)) 53 | except EOFError: 54 | break 55 | return objects 56 | 57 | 58 | def writeSplitedTraceTree(SCRIPT_DIR, contract, tx, splitedTraceTree): 59 | path = SCRIPT_DIR + "/../cache/" + contract + "_SplitedTraceTree/{}.pickle".format(tx) 60 | # print("write", path) 61 | # print(dataSourceMapList) 62 | with open(path, 'wb') as f: 63 | pickle.dump(splitedTraceTree, f) 64 | 65 | def readSplitedTraceTree(SCRIPT_DIR, contract, tx): 66 | path = SCRIPT_DIR + "/../cache/" + contract + "_SplitedTraceTree/{}.pickle".format(tx) 67 | 68 | # print("read from ", path) 69 | objects = [] 70 | # check if file exists 71 | if not os.path.exists(path): 72 | return objects 73 | with (open(path, "rb")) as openfile: 74 | while True: 75 | try: 76 | objects.append(pickle.load(openfile)) 77 | except EOFError: 78 | break 79 | return objects 80 | 81 | 82 | 83 | def writeList(path, txList): 84 | # print("write", path) 85 | # print(dataSourceMapList) 86 | with open(path, 'wb') as f: 87 | pickle.dump(txList, f) 88 | 89 | 90 | def readList(path): 91 | # path = SCRIPT_DIR + "/../cache/" + contract + "/{}.pickle".format(tx) 92 | objects = [] 93 | # check if file exists 94 | if not os.path.exists(path): 95 | return objects 96 | with (open(path, "rb")) as openfile: 97 | while True: 98 | try: 99 | objects.append(pickle.load(openfile)) 100 | except EOFError: 101 | break 102 | return objects 103 | 104 | 105 | def writeListTxt(path, txList): 106 | # allow overwrite 107 | with open(path, 'w') as f: 108 | for tx in txList: 109 | f.write(tx + "\n") 110 | 111 | 112 | def readListTxt(path): 113 | with open(path, 'r') as f: 114 | txList = f.readlines() 115 | txList = [tx.strip() for tx in txList] 116 | return txList 117 | 118 | 119 | 120 | def writeCompressedJson(filePath: str, data: dict): 121 | with gzip.open(filePath, "wb") as f: 122 | pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL) 123 | 124 | def writeJson(filePath: str, data: dict): 125 | with open(filePath, "w") as f: 126 | json.dump(data, f, indent=4) 127 | 128 | def readCompressedJson(filePath: str) -> dict: 129 | # print("readCompressedJson: ", filePath) 130 | with gzip.open(filePath, "rb") as f: 131 | jsonDict = pickle.load(f) 132 | return jsonDict 133 | 134 | def readJson(filePath: str) -> dict: 135 | with open(filePath, "r") as f: 136 | jsonDict = json.load(f) 137 | return jsonDict 138 | 139 | def setUpDirectories(script_dir , contract): 140 | path1 = script_dir + "/cache/" + contract 141 | path2 = script_dir + "/cache/" + contract + "_Access" 142 | path3 = script_dir + "/cache/" + contract + "_SplitedTraceTree" 143 | for path in [path1, path2, path3]: 144 | if not os.path.exists(path): 145 | os.makedirs(path) 146 | 147 | 148 | if __name__ == "__main__": 149 | # filePath = "/home/zhiychen/Documents/TxGuard/Benchmarks/CVEAccessControl/Txs/0x4b89f8996892d137c3de1312d1dd4e4f4ffca171/0x0ba17dc46e3a67796376e707c618898e6e1a8d163988af5acbdb6012e7e36dd1.json.gz" 150 | # temp = readCompressedJson(filePath) 151 | 152 | # # originally, pc, op, gas, stack 153 | # # batch, op, gas, depth, stack 154 | # print(temp) 155 | 156 | filePath = "/home/zhiychen/Documents/TxGuard/tempppp.txt" 157 | 158 | TxList = ["0x1", "0x2", "0x3"] 159 | writeListTxt(filePath, TxList) 160 | 161 | TxList = readListTxt(filePath) 162 | print(TxList) -------------------------------------------------------------------------------- /utilsPackage/tomlHandler.py: -------------------------------------------------------------------------------- 1 | import tomlkit 2 | 3 | 4 | settings = None 5 | with open("settings.toml", "r") as f: 6 | toml_content = f.read() 7 | settings = tomlkit.parse(toml_content) 8 | 9 | import os 10 | import sys 11 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 12 | sys.path.append(os.path.dirname(SCRIPT_DIR)) 13 | import time 14 | 15 | def changeSettings(category: str, key: str, value): 16 | settings[category][key] = value 17 | with open("settings.toml", "w") as f: 18 | f.write(tomlkit.dumps(settings)) 19 | 20 | def changeLoggingUpperBound(value): 21 | changeSettings("runtime", "LoggingUpperBound", value) 22 | time.sleep(1) 23 | 24 | if __name__ == "__main__": 25 | # changeSettings("runtime", "LoggingUpperBound", 5) 26 | changeLoggingUpperBound(6) 27 | --------------------------------------------------------------------------------