├── .gitattributes ├── .github └── workflows │ └── test_challenge_simple.yml ├── .gitignore ├── .gitmodules ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── contracts ├── Challenge.sol ├── MIPS.sol ├── MIPSMemory.sol └── lib │ ├── Lib_BytesUtils.sol │ ├── Lib_Keccak256.sol │ ├── Lib_MerkleTrie.sol │ ├── Lib_RLPReader.sol │ └── Lib_RLPWriter.sol ├── demo └── challenge_simple.sh ├── docs ├── OPML.md ├── assets │ ├── computation_graph.png │ ├── dispute.png │ ├── image.png │ └── multi-phase.png └── tutorial.md ├── hardhat.config.js ├── mlvm ├── go.mod ├── go.sum ├── main.go └── vm │ ├── ml.go │ ├── ml_test.go │ ├── run_mlgo_test.go │ ├── run_unicorn.go │ ├── trie.go │ ├── utils.go │ ├── vm.go │ └── vm_test.go ├── package-lock.json ├── package.json ├── pnpm-lock.yaml ├── scripts ├── assert.js ├── challenge.js ├── deploy.js ├── lib.js └── respond.js └── test ├── challenge_test.js ├── libkeccak.js ├── mips_test_execwtrie.js ├── mips_test_execwtrie_dynamic.js ├── mips_test_memory.js └── mips_test_oracle.js /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.github/workflows/test_challenge_simple.yml: -------------------------------------------------------------------------------- 1 | name: Test Challenge Simple 2 | 3 | on: pull_request 4 | 5 | jobs: 6 | unit: 7 | name: Simple challenge scenario 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Checkout Code 11 | uses: actions/checkout@v3 12 | with: 13 | submodules: true 14 | - name: Install Toolchain 15 | run: | 16 | sudo apt-get update 17 | sudo apt-get -y --no-install-recommends install golang-1.17 nodejs make cmake pkg-config 18 | npm install --global pnpm 19 | - name: Build 20 | run: make build 21 | - name: Run simple challenge scenario 22 | run: bash demo/challenge_simple.sh 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | artifacts 3 | cache 4 | .*.swp 5 | venv 6 | .idea 7 | *.log 8 | 9 | mlvm/mlvm -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "unicorn"] 2 | path = unicorn 3 | url = git@github.com:geohot/unicorn.git 4 | [submodule "mlgo"] 5 | path = mlgo 6 | url = git@github.com:OPML-Labs/mlgo.git 7 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # How to run instructions: 2 | # 1. Generate ssh command: ssh-keygen -t rsa -b 4096 -C "your_email@example.com" 3 | # - Save the key in local repo where Dockerfile is placed as id_rsa 4 | # - Add the public key to the GitHub account 5 | # 2. Build docker image: docker build -t ubuntu-opml-dev . 6 | # 3. Run the hardhat: docker run -it --rm --name ubuntu-opml-dev-container ubuntu-opml-dev bash -c "npx hardhat node" 7 | # 4. Run the challange script on the same container: docker exec -it ubuntu-opml-dev-container bash -c "./demo/challenge_simple.sh" 8 | 9 | 10 | # Use an official Ubuntu as a parent image 11 | FROM ubuntu:22.04 12 | 13 | # Set environment variables to non-interactive to avoid prompts during package installations 14 | ENV DEBIAN_FRONTEND=noninteractive 15 | 16 | # Update the package list and install dependencies 17 | RUN apt-get update && apt-get install -y \ 18 | build-essential \ 19 | cmake \ 20 | git \ 21 | golang \ 22 | wget \ 23 | curl \ 24 | python3 \ 25 | python3-pip \ 26 | python3-venv \ 27 | unzip \ 28 | file \ 29 | openssh-client \ 30 | && apt-get clean \ 31 | && rm -rf /var/lib/apt/lists/* 32 | 33 | # Install Node.js and npm 34 | RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - && \ 35 | apt-get install -y nodejs 36 | 37 | # Copy SSH keys into the container 38 | COPY id_rsa /root/.ssh/id_rsa 39 | RUN chmod 600 /root/.ssh/id_rsa 40 | # Configure SSH to skip host key verification 41 | RUN echo "Host *\n\tStrictHostKeyChecking no\n" >> /root/.ssh/config 42 | 43 | # Set the working directory 44 | WORKDIR /root 45 | 46 | # Clone the OPML repository 47 | RUN git clone git@github.com:ora-io/opml.git --recursive 48 | WORKDIR /root/opml 49 | 50 | # Build the OPML project 51 | RUN make build 52 | 53 | # Change permission for the challenge script 54 | RUN chmod +x demo/challenge_simple.sh 55 | 56 | # Default command 57 | CMD ["bash"] 58 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Optimism 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | 3 | build: submodules libunicorn mlvm contracts mlgo 4 | .PHONY: build 5 | 6 | submodules: 7 | # CI will checkout submodules on its own (and fails on these commands) 8 | if [[ -z "$$GITHUB_ENV" ]]; then \ 9 | git submodule init; \ 10 | git submodule update; \ 11 | fi 12 | .PHONY: submodules 13 | 14 | # Approximation, use `make libunicorn_rebuild` to force. 15 | unicorn/build: unicorn/CMakeLists.txt 16 | mkdir -p unicorn/build 17 | cd unicorn/build && cmake .. -DUNICORN_ARCH=mips -DCMAKE_BUILD_TYPE=Release 18 | # Not sure why, but the second invocation is needed for fresh installs on MacOS. 19 | if [ "$(shell uname)" == "Darwin" ]; then \ 20 | cd unicorn/build && cmake .. -DUNICORN_ARCH=mips -DCMAKE_BUILD_TYPE=Release; \ 21 | fi 22 | 23 | # Rebuild whenever anything in the unicorn/ directory changes. 24 | unicorn/build/libunicorn.so: unicorn/build unicorn 25 | cd unicorn/build && make -j8 26 | # The Go linker / runtime expects dynamic libraries in the unicorn/ dir. 27 | find ./unicorn/build -name "libunicorn.*" | xargs -L 1 -I {} cp {} ./unicorn/ 28 | # Update timestamp on libunicorn.so to make it more recent than the build/ dir. 29 | # On Mac this will create a new empty file (dyn libraries are .dylib), but works 30 | # fine for the purpose of avoiding recompilation. 31 | touch unicorn/build/libunicorn.so 32 | 33 | libunicorn: unicorn/build/libunicorn.so 34 | .PHONY: libunicorn 35 | 36 | libunicorn_rebuild: 37 | touch unicorn/CMakeLists.txt 38 | make libunicorn 39 | .PHONY: libunicorn_rebuild 40 | 41 | 42 | mlvm: 43 | cd mlvm && go build 44 | .PHONY: mlvm 45 | 46 | mlgo: 47 | cd mlgo && pip install -r requirements.txt 48 | cd mlgo/examples/mnist_mips && ./build.sh 49 | .PHONY: mlgo 50 | 51 | contracts: nodejs 52 | npx hardhat compile 53 | .PHONY: contracts 54 | 55 | nodejs: 56 | if [ -x "$$(command -v pnpm)" ]; then \ 57 | pnpm install; \ 58 | else \ 59 | npm install; \ 60 | fi 61 | .PHONY: nodejs 62 | 63 | # Must be a definition and not a rule, otherwise it gets only called once and 64 | # not before each test as we wish. 65 | define clear_cache 66 | rm -rf /tmp/cannon 67 | mkdir -p /tmp/cannon 68 | endef 69 | 70 | clear_cache: 71 | $(call clear_cache) 72 | .PHONY: clear_cache 73 | 74 | 75 | test_contracts: 76 | $(call clear_cache) 77 | npx hardhat test 78 | .PHONY: test_contracts 79 | 80 | 81 | clean: 82 | rm -rf artifacts 83 | rm -f unicorn/libunicorn.* 84 | .PHONY: clean 85 | 86 | mrproper: clean 87 | rm -rf cache 88 | rm -rf node_modules 89 | rm -rf unicorn/build 90 | .PHONY: mrproper 91 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OPML: OPtimistic Machine Learning on Blockchain 2 | 3 | OPML enables off-chain AI model inference using optimistic approach with an on chain interactive dispute engine implementing fault proofs. 4 | 5 | For more in-depth information, refer to the [project wiki](https://github.com/hyperoracle/opml/wiki). 6 | 7 | You can also find a tutorial on building a straightforward handwritten digit recognition DNN model (MNIST) within OPML in the [`docs/tutorial.md`](docs/tutorial.md). 8 | 9 | ## Building 10 | 11 | Pre-requisites: Go (Go 1.19), Node.js, Make, and CMake. 12 | 13 | ``` 14 | git clone git@github.com:hyperoracle/opml.git --recursive 15 | make build 16 | ``` 17 | 18 | ## Examples 19 | 20 | The script files [`demo/challenge_simple.sh`](demo/challenge_simple.sh) presents an example scenario (a DNN model for MNIST) demonstrating the whole process of a fault proof, including the challenge game and single step verification. 21 | 22 | To test the example, we should first start a local node: 23 | 24 | ```shell 25 | npx hardhat node 26 | ``` 27 | 28 | Then we can run: 29 | 30 | ```shell 31 | bash ./demo/challenge_simple.sh 32 | ``` 33 | 34 | A large language model, the llama example is provided in the branch ["llama"](https://github.com/hyperoracle/opml/tree/llama) (It also works for llama 2). 35 | 36 | ## Roadmap 37 | 38 | 🔨 = Pending 39 | 40 | 🛠 = Work In Progress 41 | 42 | ✅ = Feature complete 43 | 44 | 45 | | Feature | Status | 46 | | ------- | :------: | 47 | | **Supported Model** | | 48 | | DNN for MNIST | ✅ | 49 | | LLaMA | ✅ | 50 | | General DNN Model (Onnx Support) | 🛠 | 51 | | Traditional ML Algorithm (Decision Tree, KNN etc) | 🔨 | 52 | | **Mode** | | 53 | | Inference| ✅ | 54 | | Training | 🔨 | 55 | | Fine-tuning | 🔨 | 56 | | **Optimization** | | 57 | | ZK Fault Proof with zkOracle and zkWASM | 🛠 | 58 | | GPU Acceleration | 🛠 | 59 | | High Performance VM | 🛠 | 60 | | **Functionality** | | 61 | | User-Friendly SDK| 🛠 | 62 | 63 | ## Project Structure 64 | 65 | ``` 66 | mlgo -- A tensor library for machine learning in pure Golang that can run on MIPS. 67 | mlvm -- A MIPS runtime with ML execution 68 | contracts -- A Merkleized MIPS processor on chain + the challenge logic 69 | ``` 70 | 71 | ## License 72 | 73 | This code is MIT licensed. 74 | 75 | Part of this code is borrowed from `ethereum-optimism/cannon` 76 | 77 | Note: This code is unaudited. It in NO WAY should be used to secure any money until a lot more 78 | testing and auditing are done. 79 | -------------------------------------------------------------------------------- /contracts/Challenge.sol: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | pragma solidity ^0.7.3; 3 | pragma experimental ABIEncoderV2; 4 | 5 | import "./lib/Lib_RLPReader.sol"; 6 | 7 | /// @notice MIPS virtual machine interface 8 | interface IMIPS { 9 | /// @notice Given a MIPS state hash (includes code & registers), execute the next instruction and returns 10 | /// the update state hash. 11 | function Step(bytes32 stateHash) external returns (bytes32); 12 | 13 | /// @notice Returns the associated MIPS memory contract. 14 | function m() external pure returns (IMIPSMemory); 15 | } 16 | 17 | /// @notice MIPS memory (really "state", including registers and memory-mapped I/O) 18 | interface IMIPSMemory { 19 | /// @notice Adds a `(hash(anything) => anything)` entry to the mapping that underpins all the 20 | /// Merkle tries that this contract deals with (where "state hash" = Merkle root of such 21 | /// a trie). 22 | /// @param anything node data to add to the trie 23 | function AddTrieNode(bytes calldata anything) external; 24 | 25 | function ReadMemory(bytes32 stateHash, uint32 addr) external view returns (uint32); 26 | function ReadBytes32(bytes32 stateHash, uint32 addr) external view returns (bytes32); 27 | function ReadMemoryToBytes(bytes32 stateHash, uint32 addr) external view returns (bytes memory); 28 | 29 | /// @notice Write 32 bits at the given address and returns the updated state hash. 30 | function WriteMemory(bytes32 stateHash, uint32 addr, uint32 val) external returns (bytes32); 31 | 32 | /// @notice Write 32 bytes at the given address and returns the updated state hash. 33 | function WriteBytes32(bytes32 stateHash, uint32 addr, bytes32 val) external returns (bytes32); 34 | } 35 | 36 | /// @notice Implementation of the challenge game, which allows a challenger to challenge an L1 block 37 | /// by asserting a different state root for the transition implied by the block's 38 | /// transactions. The challenger plays against a defender (the owner of this contract), 39 | /// which we assume acts honestly. The challenger and the defender perform a binary search 40 | /// over the execution trace of the fault proof program (in this case minigeth), in order 41 | /// to determine a single execution step that they disagree on, at which point that step 42 | /// can be executed on-chain in order to determine if the challenge is valid. 43 | contract Challenge { 44 | address payable immutable owner; 45 | 46 | IMIPS immutable mips; 47 | IMIPSMemory immutable mem; 48 | 49 | /// @notice State hash of the fault proof program's initial MIPS state. 50 | bytes32 public immutable globalStartState; 51 | 52 | constructor(IMIPS _mips, bytes32 _globalStartState) { 53 | owner = msg.sender; 54 | mips = _mips; 55 | mem = _mips.m(); 56 | globalStartState = _globalStartState; 57 | } 58 | 59 | struct ChallengeData { 60 | // Left bound of the binary search: challenger & defender agree on all steps <= L. 61 | uint256 L; 62 | // Right bound of the binary search: challenger & defender disagree on all steps >= R. 63 | uint256 R; 64 | // Maps step numbers to asserted state hashes for the challenger. 65 | mapping(uint256 => bytes32) assertedState; 66 | // Maps step numbers to asserted state hashes for the defender. 67 | mapping(uint256 => bytes32) defendedState; 68 | // Address of the challenger. 69 | address payable challenger; 70 | // Block number preceding the challenged block. 71 | uint256 blockNumberN; 72 | } 73 | 74 | /// @notice ID if the last created challenged, incremented for new challenge IDs. 75 | uint256 public lastChallengeId = 0; 76 | 77 | /// @notice Maps challenge IDs to challenge data. 78 | mapping(uint256 => ChallengeData) public challenges; 79 | 80 | /// @notice Emitted when a new challenge is created. 81 | event ChallengeCreated(uint256 challengeId); 82 | 83 | 84 | /// @notice proposer's results 85 | bytes public proposedResults; 86 | 87 | /// @notice challenger's results 88 | bytes public challengerResults; 89 | 90 | /// @notice Proposer should first upload the results and stake some money, waiting for the challenge 91 | /// process. Note that the results can only be set once (TODO) 92 | function uploadResult(bytes calldata data) public { 93 | require(data.length % 32 == 0, "the result should 32-align"); 94 | proposedResults = data; 95 | } 96 | 97 | 98 | /// @notice Challenges the pure computation without accessing to the blockchain data 99 | /// Before calling this, it is necessary to have loaded all the trie node necessary to 100 | /// write the input hash in the Merkleized initial MIPS state, and to read the output hash 101 | /// and machine state from the Merkleized final MIPS state (i.e. `finalSystemState`). Use 102 | /// `MIPSMemory.AddTrieNode` for this purpose. Use `callWithTrieNodes` to figure out 103 | /// which nodes you need. 104 | /// @param finalSystemState The state hash of the fault proof program's final MIPS state. 105 | /// @param stepCount The number of steps (MIPS instructions) taken to execute the fault proof 106 | /// program. 107 | /// @return The challenge identifier 108 | function initiatePureComputationChallenge( 109 | bytes32 finalSystemState, uint256 stepCount) 110 | external 111 | returns (uint256) 112 | { 113 | // Write input hash at predefined memory address. 114 | bytes32 startState = globalStartState; 115 | 116 | // Confirm that `finalSystemState` asserts the state you claim and that the machine is stopped. 117 | require(mem.ReadMemory(finalSystemState, 0xC0000080) == 0x5EAD0000, 118 | "the final MIPS machine state is not stopped (PC != 0x5EAD0000)"); 119 | 120 | // maybe we do not need that, since it is binded with evm smart contract? 121 | // require(mem.ReadMemory(finalSystemState, 0x30000800) == 0x1337f00d, 122 | // "the final state root has not been written a the predefined MIPS memory location"); 123 | 124 | // the challenger should upload his results on chain 125 | // for a valid challenge, the challenge results != proposer results! 126 | bytes memory result = mem.ReadMemoryToBytes(finalSystemState, 0x32000000); 127 | // require(keccak256(result) != keccak256(proposedResults), "the challenger's results should be different from the proposed results"); 128 | challengerResults = result; 129 | 130 | uint256 challengeId = lastChallengeId++; 131 | ChallengeData storage c = challenges[challengeId]; 132 | 133 | // A NEW CHALLENGER APPEARS 134 | c.challenger = msg.sender; 135 | // c.blockNumberN = blockNumberN; // no need to set the blockNumber 136 | c.assertedState[0] = startState; 137 | c.defendedState[0] = startState; 138 | c.assertedState[stepCount] = finalSystemState; 139 | c.L = 0; 140 | c.R = stepCount; 141 | 142 | emit ChallengeCreated(challengeId); 143 | return challengeId; 144 | } 145 | 146 | 147 | /// @notice Calling `initiateChallenge`, `confirmStateTransition` or `denyStateTransition requires 148 | /// some trie nodes to have been supplied beforehand (see these functions for details). 149 | /// This function can be used to figure out which nodes are needed, as memory-accessing 150 | /// functions in MIPSMemory.sol will revert with the missing node ID when a node is 151 | /// missing. Therefore, you can call this function repeatedly via `eth_call`, and 152 | /// iteratively build the list of required node until the call succeeds. 153 | /// @param target The contract to call to (usually this contract) 154 | /// @param dat The data to include in the call (usually the calldata for a call to 155 | /// one of the aforementionned functions) 156 | /// @param nodes The nodes to add the MIPS state trie before making the call 157 | function callWithTrieNodes(address target, bytes calldata dat, bytes[] calldata nodes) public { 158 | for (uint i = 0; i < nodes.length; i++) { 159 | mem.AddTrieNode(nodes[i]); 160 | } 161 | (bool success, bytes memory revertData) = target.call(dat); 162 | if (!success) { 163 | uint256 revertDataLength = revertData.length; 164 | assembly { 165 | let revertDataStart := add(revertData, 32) 166 | revert(revertDataStart, revertDataLength) 167 | } 168 | } 169 | } 170 | 171 | /// @notice Indicates whether the given challenge is still searching (true), or if the single step 172 | /// of disagreement has been found (false). 173 | function isSearching(uint256 challengeId) view public returns (bool) { 174 | ChallengeData storage c = challenges[challengeId]; 175 | require(c.challenger != address(0), "invalid challenge"); 176 | return c.L + 1 != c.R; 177 | } 178 | 179 | /// @notice Returns the next step number where the challenger and the defender must compared 180 | /// state hash, namely the midpoint between the current left and right bounds of the 181 | /// binary search. 182 | function getStepNumber(uint256 challengeId) view public returns (uint256) { 183 | ChallengeData storage c = challenges[challengeId]; 184 | require(c.challenger != address(0), "invalid challenge"); 185 | return (c.L+c.R)/2; 186 | } 187 | 188 | /// @notice Returns the last state hash proposed by the challenger during the binary search. 189 | function getProposedState(uint256 challengeId) view public returns (bytes32) { 190 | ChallengeData storage c = challenges[challengeId]; 191 | require(c.challenger != address(0), "invalid challenge"); 192 | uint256 stepNumber = getStepNumber(challengeId); 193 | return c.assertedState[stepNumber]; 194 | } 195 | 196 | /// @notice The challenger can call this function to submit the state hash for the next step 197 | /// in the binary search (cf. `getStepNumber`). 198 | function proposeState(uint256 challengeId, bytes32 stateHash) external { 199 | ChallengeData storage c = challenges[challengeId]; 200 | require(c.challenger != address(0), "invalid challenge"); 201 | require(c.challenger == msg.sender, "must be challenger"); 202 | require(isSearching(challengeId), "must be searching"); 203 | 204 | uint256 stepNumber = getStepNumber(challengeId); 205 | require(c.assertedState[stepNumber] == bytes32(0), "state already proposed"); 206 | c.assertedState[stepNumber] = stateHash; 207 | } 208 | 209 | /// @notice The defender can call this function to submit the state hash for the next step 210 | /// in the binary search (cf. `getStepNumber`). He can only do this after the challenger 211 | /// has submitted his own state hash for this step. 212 | /// If the defender believes there are less steps in the execution of the fault proof 213 | /// program than the current step number, he should submit the final state hash. 214 | function respondState(uint256 challengeId, bytes32 stateHash) external { 215 | ChallengeData storage c = challenges[challengeId]; 216 | require(c.challenger != address(0), "invalid challenge"); 217 | require(owner == msg.sender, "must be owner"); 218 | require(isSearching(challengeId), "must be searching"); 219 | 220 | uint256 stepNumber = getStepNumber(challengeId); 221 | require(c.assertedState[stepNumber] != bytes32(0), "challenger state not proposed"); 222 | require(c.defendedState[stepNumber] == bytes32(0), "state already proposed"); 223 | 224 | // Technically, we don't have to save these states, but we have to if we want to let the 225 | // defender terminate the proof early (and not via a timeout) after the binary search completes. 226 | c.defendedState[stepNumber] = stateHash; 227 | 228 | // update binary search bounds 229 | if (c.assertedState[stepNumber] == c.defendedState[stepNumber]) { 230 | c.L = stepNumber; // agree 231 | } else { 232 | c.R = stepNumber; // disagree 233 | } 234 | } 235 | 236 | /// @notice Emitted when the challenger can provably be shown to be correct about his assertion. 237 | event ChallengerWins(uint256 challengeId); 238 | 239 | /// @notice Emitted when the challenger can provably be shown to be wrong about his assertion. 240 | event ChallengerLoses(uint256 challengeId); 241 | 242 | /// @notice Emitted when the challenger should lose if he does not generate a `ChallengerWins` 243 | /// event in a timely manner (TBD). This occurs in a specific scenario when we can't 244 | /// explicitly verify that the defender is right (cf. `denyStateTransition). 245 | event ChallengerLosesByDefault(uint256 challengeId); 246 | 247 | /// @notice Anybody can call this function to confirm that the single execution step that the 248 | /// challenger and defender disagree on does indeed yield the result asserted by the 249 | /// challenger, leading to him winning the challenge. 250 | /// Before calling this function, you need to add trie nodes so that the MIPS state can be 251 | /// read/written by the single step execution. Use `MIPSMemory.AddTrieNode` for this 252 | /// purpose. Use `callWithTrieNodes` to figure out which nodes you need. 253 | /// You will also need to supply any preimage that the step tries to access with 254 | /// `MIPSMemory.AddPreimage`. See `scripts/assert.js` for details on how this can be 255 | /// done. 256 | function confirmStateTransition(uint256 challengeId) external { 257 | ChallengeData storage c = challenges[challengeId]; 258 | require(c.challenger != address(0), "invalid challenge"); 259 | require(!isSearching(challengeId), "binary search not finished"); 260 | 261 | bytes32 stepState = mips.Step(c.assertedState[c.L]); 262 | require(stepState == c.assertedState[c.R], "wrong asserted state for challenger"); 263 | 264 | // pay out bounty!! 265 | (bool sent, ) = c.challenger.call{value: address(this).balance}(""); 266 | require(sent, "Failed to send Ether"); 267 | 268 | emit ChallengerWins(challengeId); 269 | } 270 | 271 | /// @notice Anybody can call this function to confirm that the single execution step that the 272 | /// challenger and defender disagree on does indeed yield the result asserted by the 273 | /// defender, leading to the challenger losing the challenge. 274 | /// Before calling this function, you need to add trie nodes so that the MIPS state can be 275 | /// read/written by the single step execution. Use `MIPSMemory.AddTrieNode` for this 276 | /// purpose. Use `callWithTrieNodes` to figure out which nodes you need. 277 | /// You will also need to supply any preimage that the step tries to access with 278 | /// `MIPSMemory.AddPreimage`. See `scripts/assert.js` for details on how this can be 279 | /// done. 280 | function denyStateTransition(uint256 challengeId) external { 281 | ChallengeData storage c = challenges[challengeId]; 282 | require(c.challenger != address(0), "invalid challenge"); 283 | require(!isSearching(challengeId), "binary search not finished"); 284 | 285 | // We run this before the next check so that if executing the final step somehow 286 | // causes a revert, then at least we do not emit `ChallengerLosesByDefault` when we know that 287 | // the challenger can't win (even if right) because of the revert. 288 | bytes32 stepState = mips.Step(c.defendedState[c.L]); 289 | 290 | // If the challenger always agrees with the defender during the search, we end up with: 291 | // c.L + 1 == c.R == stepCount (from `initiateChallenge`) 292 | // In this case, the defender didn't assert his state hash for c.R, which makes 293 | // `c.defendedState[c.R]` zero. This means we can't verify that the defender right about the 294 | // final execution step. 295 | // The solution is to emit `ChallengerLosesByDefault` to signify the challenger should lose 296 | // if he can't emit `ChallengerWins` in a timely manner. 297 | if (c.defendedState[c.R] == bytes32(0)) { 298 | emit ChallengerLosesByDefault(challengeId); 299 | return; 300 | } 301 | 302 | require(stepState == c.defendedState[c.R], "wrong asserted state for defender"); 303 | 304 | // consider the challenger mocked 305 | emit ChallengerLoses(challengeId); 306 | } 307 | 308 | /// @notice Allow sending money to the contract (without calldata). 309 | receive() external payable {} 310 | 311 | /// @notice Allows the owner to withdraw funds from the contract. 312 | function withdraw() external { 313 | require(msg.sender == owner); 314 | owner.transfer(address(this).balance); 315 | } 316 | } 317 | -------------------------------------------------------------------------------- /contracts/MIPS.sol: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | pragma solidity ^0.7.3; 3 | import "./MIPSMemory.sol"; 4 | 5 | // https://inst.eecs.berkeley.edu/~cs61c/resources/MIPS_Green_Sheet.pdf 6 | // https://uweb.engr.arizona.edu/~ece369/Resources/spim/MIPSReference.pdf 7 | // https://en.wikibooks.org/wiki/MIPS_Assembly/Instruction_Formats 8 | 9 | // https://www.cs.cmu.edu/afs/cs/academic/class/15740-f97/public/doc/mips-isa.pdf 10 | // page A-177 11 | 12 | // This is a separate contract from the challenge contract 13 | // Anyone can use it to validate a MIPS state transition 14 | // First, to prepare, you call AddMerkleState, which adds valid state nodes in the stateHash. 15 | // If you are using the Preimage oracle, you call AddPreimage 16 | // Then, you call Step. Step will revert if state is missing. If all state is present, it will return the next hash 17 | 18 | contract MIPS { 19 | MIPSMemory public immutable m; 20 | 21 | uint32 constant public REG_OFFSET = 0xc0000000; 22 | uint32 constant public REG_ZERO = REG_OFFSET; 23 | uint32 constant public REG_LR = REG_OFFSET + 0x1f*4; 24 | uint32 constant public REG_PC = REG_OFFSET + 0x20*4; 25 | uint32 constant public REG_HI = REG_OFFSET + 0x21*4; 26 | uint32 constant public REG_LO = REG_OFFSET + 0x22*4; 27 | uint32 constant public REG_HEAP = REG_OFFSET + 0x23*4; 28 | 29 | uint32 constant public HEAP_START = 0x20000000; 30 | uint32 constant public BRK_START = 0x40000000; 31 | 32 | constructor() { 33 | m = new MIPSMemory(); 34 | } 35 | 36 | bool constant public debug = true; 37 | 38 | event DidStep(bytes32 stateHash); 39 | event DidWriteMemory(uint32 addr, uint32 value); 40 | event TryReadMemory(uint32 addr); 41 | event DidReadMemory(uint32 addr, uint32 value); 42 | 43 | function WriteMemory(bytes32 stateHash, uint32 addr, uint32 value) internal returns (bytes32) { 44 | if (address(m) != address(0)) { 45 | emit DidWriteMemory(addr, value); 46 | bytes32 newStateHash = m.WriteMemory(stateHash, addr, value); 47 | require(m.ReadMemory(newStateHash, addr) == value, "memory readback check failed"); 48 | return newStateHash; 49 | } 50 | assembly { 51 | // TODO: this is actually doing an SLOAD first 52 | sstore(addr, value) 53 | } 54 | return stateHash; 55 | } 56 | 57 | function ReadMemory(bytes32 stateHash, uint32 addr) internal returns (uint32 ret) { 58 | if (address(m) != address(0)) { 59 | emit TryReadMemory(addr); 60 | ret = m.ReadMemory(stateHash, addr); 61 | //emit DidReadMemory(addr, ret); 62 | return ret; 63 | } 64 | assembly { 65 | ret := sload(addr) 66 | } 67 | } 68 | 69 | function Steps(bytes32 stateHash, uint count) public returns (bytes32) { 70 | for (uint i = 0; i < count; i++) { 71 | stateHash = Step(stateHash); 72 | } 73 | return stateHash; 74 | } 75 | 76 | function SE(uint32 dat, uint32 idx) internal pure returns (uint32) { 77 | bool isSigned = (dat >> (idx-1)) != 0; 78 | uint256 signed = ((1 << (32-idx)) - 1) << idx; 79 | uint256 mask = (1 << idx) - 1; 80 | return uint32(dat&mask | (isSigned ? signed : 0)); 81 | } 82 | 83 | function handleSyscall(bytes32 stateHash) internal returns (bytes32, bool) { 84 | uint32 syscall_no = ReadMemory(stateHash, REG_OFFSET+2*4); 85 | uint32 v0 = 0; 86 | bool exit = false; 87 | 88 | if (syscall_no == 4090) { 89 | // mmap 90 | uint32 a0 = ReadMemory(stateHash, REG_OFFSET+4*4); 91 | if (a0 == 0) { 92 | uint32 sz = ReadMemory(stateHash, REG_OFFSET+5*4); 93 | uint32 hr = ReadMemory(stateHash, REG_HEAP); 94 | v0 = HEAP_START + hr; 95 | stateHash = WriteMemory(stateHash, REG_HEAP, hr+sz); 96 | } else { 97 | v0 = a0; 98 | } 99 | } else if (syscall_no == 4045) { 100 | // brk 101 | v0 = BRK_START; 102 | } else if (syscall_no == 4120) { 103 | // clone (not supported) 104 | v0 = 1; 105 | } else if (syscall_no == 4246) { 106 | // exit group 107 | exit = true; 108 | } 109 | 110 | stateHash = WriteMemory(stateHash, REG_OFFSET+2*4, v0); 111 | stateHash = WriteMemory(stateHash, REG_OFFSET+7*4, 0); 112 | return (stateHash, exit); 113 | } 114 | 115 | function Step(bytes32 stateHash) public returns (bytes32 newStateHash) { 116 | uint32 pc = ReadMemory(stateHash, REG_PC); 117 | if (pc == 0x5ead0000) { 118 | return stateHash; 119 | } 120 | newStateHash = stepPC(stateHash, pc, pc+4); 121 | if (address(m) != address(0)) { 122 | emit DidStep(newStateHash); 123 | } 124 | } 125 | 126 | // will revert if any required input state is missing 127 | function stepPC(bytes32 stateHash, uint32 pc, uint32 nextPC) internal returns (bytes32) { 128 | // instruction fetch 129 | uint32 insn = ReadMemory(stateHash, pc); 130 | 131 | uint32 opcode = insn >> 26; // 6-bits 132 | uint32 func = insn & 0x3f; // 6-bits 133 | 134 | // j-type j/jal 135 | if (opcode == 2 || opcode == 3) { 136 | stateHash = stepPC(stateHash, nextPC, 137 | SE(insn&0x03FFFFFF, 26) << 2); 138 | if (opcode == 3) { 139 | stateHash = WriteMemory(stateHash, REG_LR, pc+8); 140 | } 141 | return stateHash; 142 | } 143 | 144 | // register fetch 145 | uint32 storeAddr = REG_ZERO; 146 | uint32 rs; 147 | uint32 rt; 148 | uint32 rtReg = REG_OFFSET + ((insn >> 14) & 0x7C); 149 | 150 | // R-type or I-type (stores rt) 151 | rs = ReadMemory(stateHash, REG_OFFSET + ((insn >> 19) & 0x7C)); 152 | storeAddr = REG_OFFSET + ((insn >> 14) & 0x7C); 153 | if (opcode == 0 || opcode == 0x1c) { 154 | // R-type (stores rd) 155 | rt = ReadMemory(stateHash, rtReg); 156 | storeAddr = REG_OFFSET + ((insn >> 9) & 0x7C); 157 | } else if (opcode < 0x20) { 158 | // rt is SignExtImm 159 | // don't sign extend for andi, ori, xori 160 | if (opcode == 0xC || opcode == 0xD || opcode == 0xe) { 161 | // ZeroExtImm 162 | rt = insn&0xFFFF; 163 | } else { 164 | // SignExtImm 165 | rt = SE(insn&0xFFFF, 16); 166 | } 167 | } else if (opcode >= 0x28 || opcode == 0x22 || opcode == 0x26) { 168 | // store rt value with store 169 | rt = ReadMemory(stateHash, rtReg); 170 | 171 | // store actual rt with lwl and lwr 172 | storeAddr = rtReg; 173 | } 174 | 175 | if ((opcode >= 4 && opcode < 8) || opcode == 1) { 176 | bool shouldBranch = false; 177 | 178 | if (opcode == 4 || opcode == 5) { // beq/bne 179 | rt = ReadMemory(stateHash, rtReg); 180 | shouldBranch = (rs == rt && opcode == 4) || (rs != rt && opcode == 5); 181 | } else if (opcode == 6) { shouldBranch = int32(rs) <= 0; // blez 182 | } else if (opcode == 7) { shouldBranch = int32(rs) > 0; // bgtz 183 | } else if (opcode == 1) { 184 | // regimm 185 | uint32 rtv = ((insn >> 16) & 0x1F); 186 | if (rtv == 0) shouldBranch = int32(rs) < 0; // bltz 187 | if (rtv == 1) shouldBranch = int32(rs) >= 0; // bgez 188 | } 189 | 190 | if (shouldBranch) { 191 | return stepPC(stateHash, nextPC, 192 | pc + 4 + (SE(insn&0xFFFF, 16)<<2)); 193 | } 194 | // branch not taken 195 | return stepPC(stateHash, nextPC, nextPC+4); 196 | } 197 | 198 | // memory fetch (all I-type) 199 | // we do the load for stores also 200 | uint32 mem; 201 | if (opcode >= 0x20) { 202 | // M[R[rs]+SignExtImm] 203 | uint32 SignExtImm = SE(insn&0xFFFF, 16); 204 | rs += SignExtImm; 205 | uint32 addr = rs & 0xFFFFFFFC; 206 | mem = ReadMemory(stateHash, addr); 207 | if (opcode >= 0x28 && opcode != 0x30) { 208 | // store 209 | storeAddr = addr; 210 | } 211 | } 212 | 213 | // ALU 214 | uint32 val = execute(insn, rs, rt, mem); 215 | 216 | if (opcode == 0 && func >= 8 && func < 0x1c) { 217 | if (func == 8 || func == 9) { 218 | // jr/jalr 219 | stateHash = stepPC(stateHash, nextPC, rs); 220 | if (func == 9) { 221 | stateHash = WriteMemory(stateHash, REG_LR, pc+8); 222 | } 223 | return stateHash; 224 | } 225 | 226 | // handle movz and movn when they don't write back 227 | if (func == 0xa && rt != 0) { // movz 228 | storeAddr = REG_ZERO; 229 | } 230 | if (func == 0xb && rt == 0) { // movn 231 | storeAddr = REG_ZERO; 232 | } 233 | 234 | // syscall (can read and write) 235 | if (func == 0xC) { 236 | //revert("unhandled syscall"); 237 | bool exit; 238 | (stateHash, exit) = handleSyscall(stateHash); 239 | if (exit) { 240 | nextPC = 0x5ead0000; 241 | } 242 | } 243 | 244 | // lo and hi registers 245 | // can write back 246 | if (func >= 0x10 && func < 0x1c) { 247 | if (func == 0x10) val = ReadMemory(stateHash, REG_HI); // mfhi 248 | else if (func == 0x11) storeAddr = REG_HI; // mthi 249 | else if (func == 0x12) val = ReadMemory(stateHash, REG_LO); // mflo 250 | else if (func == 0x13) storeAddr = REG_LO; // mtlo 251 | 252 | uint32 hi; 253 | if (func == 0x18) { // mult 254 | uint64 acc = uint64(int64(int32(rs))*int64(int32(rt))); 255 | hi = uint32(acc>>32); 256 | val = uint32(acc); 257 | } else if (func == 0x19) { // multu 258 | uint64 acc = uint64(uint64(rs)*uint64(rt)); 259 | hi = uint32(acc>>32); 260 | val = uint32(acc); 261 | } else if (func == 0x1a) { // div 262 | hi = uint32(int32(rs)%int32(rt)); 263 | val = uint32(int32(rs)/int32(rt)); 264 | } else if (func == 0x1b) { // divu 265 | hi = rs%rt; 266 | val = rs/rt; 267 | } 268 | 269 | // lo/hi writeback 270 | if (func >= 0x18 && func < 0x1c) { 271 | stateHash = WriteMemory(stateHash, REG_HI, hi); 272 | storeAddr = REG_LO; 273 | } 274 | } 275 | } 276 | 277 | // stupid sc, write a 1 to rt 278 | if (opcode == 0x38 && rtReg != REG_ZERO) { 279 | stateHash = WriteMemory(stateHash, rtReg, 1); 280 | } 281 | 282 | // write back 283 | if (storeAddr != REG_ZERO) { 284 | stateHash = WriteMemory(stateHash, storeAddr, val); 285 | } 286 | 287 | stateHash = WriteMemory(stateHash, REG_PC, nextPC); 288 | 289 | return stateHash; 290 | } 291 | 292 | function execute(uint32 insn, uint32 rs, uint32 rt, uint32 mem) internal pure returns (uint32) { 293 | uint32 opcode = insn >> 26; // 6-bits 294 | uint32 func = insn & 0x3f; // 6-bits 295 | // TODO: deref the immed into a register 296 | 297 | if (opcode < 0x20) { 298 | // transform ArithLogI 299 | // TODO: replace with table 300 | if (opcode >= 8 && opcode < 0xF) { 301 | if (opcode == 8) { func = 0x20; } // addi 302 | else if (opcode == 9) { func = 0x21; } // addiu 303 | else if (opcode == 0xa) { func = 0x2a; } // slti 304 | else if (opcode == 0xb) { func = 0x2B; } // sltiu 305 | else if (opcode == 0xc) { func = 0x24; } // andi 306 | else if (opcode == 0xd) { func = 0x25; } // ori 307 | else if (opcode == 0xe) { func = 0x26; } // xori 308 | opcode = 0; 309 | } 310 | 311 | // 0 is opcode SPECIAL 312 | if (opcode == 0) { 313 | uint32 shamt = (insn >> 6) & 0x1f; 314 | if (func < 0x20) { 315 | if (func >= 0x08) { return rs; // jr/jalr/div + others 316 | // Shift and ShiftV 317 | } else if (func == 0x00) { return rt << shamt; // sll 318 | } else if (func == 0x02) { return rt >> shamt; // srl 319 | } else if (func == 0x03) { return SE(rt >> shamt, 32-shamt); // sra 320 | } else if (func == 0x04) { return rt << (rs&0x1F); // sllv 321 | } else if (func == 0x06) { return rt >> (rs&0x1F); // srlv 322 | } else if (func == 0x07) { return SE(rt >> rs, 32-rs); // srav 323 | } 324 | } 325 | // 0x10-0x13 = mfhi, mthi, mflo, mtlo 326 | // R-type (ArithLog) 327 | if (func == 0x20 || func == 0x21) { return rs+rt; // add or addu 328 | } else if (func == 0x22 || func == 0x23) { return rs-rt; // sub or subu 329 | } else if (func == 0x24) { return rs&rt; // and 330 | } else if (func == 0x25) { return (rs|rt); // or 331 | } else if (func == 0x26) { return (rs^rt); // xor 332 | } else if (func == 0x27) { return ~(rs|rt); // nor 333 | } else if (func == 0x2a) { 334 | return int32(rs)> (24-(rs&3)*8)) & 0xFF, 8); 349 | } else if (opcode == 0x21) { // lh 350 | return SE((mem >> (16-(rs&2)*8)) & 0xFFFF, 16); 351 | } else if (opcode == 0x22) { // lwl 352 | uint32 val = mem << ((rs&3)*8); 353 | uint32 mask = uint32(0xFFFFFFFF) << ((rs&3)*8); 354 | return (rt & ~mask) | val; 355 | } else if (opcode == 0x23) { return mem; // lw 356 | } else if (opcode == 0x24) { // lbu 357 | return (mem >> (24-(rs&3)*8)) & 0xFF; 358 | } else if (opcode == 0x25) { // lhu 359 | return (mem >> (16-(rs&2)*8)) & 0xFFFF; 360 | } else if (opcode == 0x26) { // lwr 361 | uint32 val = mem >> (24-(rs&3)*8); 362 | uint32 mask = uint32(0xFFFFFFFF) >> (24-(rs&3)*8); 363 | return (rt & ~mask) | val; 364 | } 365 | } else if (opcode == 0x28) { // sb 366 | uint32 val = (rt&0xFF) << (24-(rs&3)*8); 367 | uint32 mask = 0xFFFFFFFF ^ uint32(0xFF << (24-(rs&3)*8)); 368 | return (mem & mask) | val; 369 | } else if (opcode == 0x29) { // sh 370 | uint32 val = (rt&0xFFFF) << (16-(rs&2)*8); 371 | uint32 mask = 0xFFFFFFFF ^ uint32(0xFFFF << (16-(rs&2)*8)); 372 | return (mem & mask) | val; 373 | } else if (opcode == 0x2a) { // swl 374 | uint32 val = rt >> ((rs&3)*8); 375 | uint32 mask = uint32(0xFFFFFFFF) >> ((rs&3)*8); 376 | return (mem & ~mask) | val; 377 | } else if (opcode == 0x2b) { // sw 378 | return rt; 379 | } else if (opcode == 0x2e) { // swr 380 | uint32 val = rt << (24-(rs&3)*8); 381 | uint32 mask = uint32(0xFFFFFFFF) << (24-(rs&3)*8); 382 | return (mem & ~mask) | val; 383 | } else if (opcode == 0x30) { return mem; // ll 384 | } else if (opcode == 0x38) { return rt; // sc 385 | } 386 | 387 | revert("invalid instruction"); 388 | } 389 | } 390 | -------------------------------------------------------------------------------- /contracts/MIPSMemory.sol: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | pragma solidity ^0.7.3; 3 | 4 | import "./lib/Lib_Keccak256.sol"; 5 | import "./lib/Lib_MerkleTrie.sol"; 6 | import { Lib_BytesUtils } from "./lib/Lib_BytesUtils.sol"; 7 | 8 | contract MIPSMemory { 9 | function AddTrieNode(bytes calldata anything) public { 10 | Lib_MerkleTrie.GetTrie()[keccak256(anything)] = anything; 11 | } 12 | 13 | struct Preimage { 14 | uint64 length; 15 | mapping(uint => uint64) data; 16 | } 17 | 18 | mapping(bytes32 => Preimage) public preimage; 19 | 20 | function MissingPreimageRevert(bytes32 outhash, uint offset) internal pure { 21 | Lib_BytesUtils.revertWithHex(abi.encodePacked(outhash, offset)); 22 | } 23 | 24 | function GetPreimageLength(bytes32 outhash) public view returns (uint32) { 25 | uint64 data = preimage[outhash].length; 26 | if (data == 0) { 27 | MissingPreimageRevert(outhash, 0); 28 | } 29 | return uint32(data); 30 | } 31 | 32 | function GetPreimage(bytes32 outhash, uint offset) public view returns (uint32) { 33 | uint64 data = preimage[outhash].data[offset]; 34 | if (data == 0) { 35 | MissingPreimageRevert(outhash, offset); 36 | } 37 | return uint32(data); 38 | } 39 | 40 | function AddPreimage(bytes calldata anything, uint offset) public { 41 | require(offset & 3 == 0, "offset must be 32-bit aligned"); 42 | uint len = anything.length; 43 | require(offset < len, "offset can't be longer than input"); 44 | Preimage storage p = preimage[keccak256(anything)]; 45 | require(p.length == 0 || uint32(p.length) == len, "length is somehow wrong"); 46 | p.length = (1 << 32) | uint64(uint32(len)); 47 | p.data[offset] = (1 << 32) | 48 | ((len <= (offset+0) ? 0 : uint32(uint8(anything[offset+0]))) << 24) | 49 | ((len <= (offset+1) ? 0 : uint32(uint8(anything[offset+1]))) << 16) | 50 | ((len <= (offset+2) ? 0 : uint32(uint8(anything[offset+2]))) << 8) | 51 | ((len <= (offset+3) ? 0 : uint32(uint8(anything[offset+3]))) << 0); 52 | } 53 | 54 | // one per owner (at a time) 55 | 56 | struct LargePreimage { 57 | uint offset; 58 | uint len; 59 | uint32 data; 60 | } 61 | mapping(address => LargePreimage) public largePreimage; 62 | // sadly due to soldiity limitations this can't be in the LargePreimage struct 63 | mapping(address => uint64[25]) public largePreimageState; 64 | 65 | function AddLargePreimageInit(uint offset) public { 66 | require(offset & 3 == 0, "offset must be 32-bit aligned"); 67 | Lib_Keccak256.CTX memory c; 68 | Lib_Keccak256.keccak_init(c); 69 | largePreimageState[msg.sender] = c.A; 70 | largePreimage[msg.sender].offset = offset; 71 | largePreimage[msg.sender].len = 0; 72 | } 73 | 74 | // input 136 bytes, as many times as you'd like 75 | // Uses about 500k gas, 3435 gas/byte 76 | function AddLargePreimageUpdate(bytes calldata dat) public { 77 | require(dat.length == 136, "update must be in multiples of 136"); 78 | // sha3_process_block 79 | Lib_Keccak256.CTX memory c; 80 | c.A = largePreimageState[msg.sender]; 81 | 82 | int offset = int(largePreimage[msg.sender].offset) - int(largePreimage[msg.sender].len); 83 | if (offset >= 0 && offset < 136) { 84 | largePreimage[msg.sender].data = fbo(dat, uint(offset)); 85 | } 86 | Lib_Keccak256.sha3_xor_input(c, dat); 87 | Lib_Keccak256.sha3_permutation(c); 88 | largePreimageState[msg.sender] = c.A; 89 | largePreimage[msg.sender].len += 136; 90 | } 91 | 92 | function AddLargePreimageFinal(bytes calldata idat) public view returns (bytes32, uint32, uint32) { 93 | require(idat.length < 136, "final must be less than 136"); 94 | int offset = int(largePreimage[msg.sender].offset) - int(largePreimage[msg.sender].len); 95 | require(offset < int(idat.length), "offset must be less than length"); 96 | Lib_Keccak256.CTX memory c; 97 | c.A = largePreimageState[msg.sender]; 98 | 99 | bytes memory dat = new bytes(136); 100 | for (uint i = 0; i < idat.length; i++) { 101 | dat[i] = idat[i]; 102 | } 103 | uint len = largePreimage[msg.sender].len + idat.length; 104 | uint32 data = largePreimage[msg.sender].data; 105 | if (offset >= 0) { 106 | data = fbo(dat, uint(offset)); 107 | } 108 | dat[135] = bytes1(uint8(0x80)); 109 | dat[idat.length] |= bytes1(uint8(0x1)); 110 | 111 | Lib_Keccak256.sha3_xor_input(c, dat); 112 | Lib_Keccak256.sha3_permutation(c); 113 | 114 | bytes32 outhash = Lib_Keccak256.get_hash(c); 115 | require(len < 0x10000000, "max length is 32-bit"); 116 | return (outhash, uint32(len), data); 117 | } 118 | 119 | function AddLargePreimageFinalSaved(bytes calldata idat) public { 120 | bytes32 outhash; 121 | uint32 len; 122 | uint32 data; 123 | (outhash, len, data) = AddLargePreimageFinal(idat); 124 | 125 | Preimage storage p = preimage[outhash]; 126 | require(p.length == 0 || uint32(p.length) == len, "length is somehow wrong"); 127 | require(largePreimage[msg.sender].offset < len, "offset is somehow beyond length"); 128 | p.length = (1 << 32) | uint64(len); 129 | p.data[largePreimage[msg.sender].offset] = (1 << 32) | data; 130 | } 131 | 132 | function tb(uint32 dat) internal pure returns (bytes memory) { 133 | bytes memory ret = new bytes(4); 134 | ret[0] = bytes1(uint8(dat >> 24)); 135 | ret[1] = bytes1(uint8(dat >> 16)); 136 | ret[2] = bytes1(uint8(dat >> 8)); 137 | ret[3] = bytes1(uint8(dat >> 0)); 138 | return ret; 139 | } 140 | 141 | function fb(bytes memory dat) internal pure returns (uint32) { 142 | require(dat.length == 4, "wrong length value"); 143 | uint32 ret = uint32(uint8(dat[0])) << 24 | 144 | uint32(uint8(dat[1])) << 16 | 145 | uint32(uint8(dat[2])) << 8 | 146 | uint32(uint8(dat[3])); 147 | return ret; 148 | } 149 | 150 | function fbo(bytes memory dat, uint offset) internal pure returns (uint32) { 151 | uint32 ret = uint32(uint8(dat[offset+0])) << 24 | 152 | uint32(uint8(dat[offset+1])) << 16 | 153 | uint32(uint8(dat[offset+2])) << 8 | 154 | uint32(uint8(dat[offset+3])); 155 | return ret; 156 | } 157 | 158 | function WriteMemory(bytes32 stateHash, uint32 addr, uint32 value) public returns (bytes32) { 159 | require(addr & 3 == 0, "write memory must be 32-bit aligned"); 160 | return Lib_MerkleTrie.update(tb(addr>>2), tb(value), stateHash); 161 | } 162 | 163 | function WriteBytes32(bytes32 stateHash, uint32 addr, bytes32 val) public returns (bytes32) { 164 | for (uint32 i = 0; i < 32; i += 4) { 165 | uint256 tv = uint256(val>>(224-(i*8))); 166 | stateHash = WriteMemory(stateHash, addr+i, uint32(tv)); 167 | } 168 | return stateHash; 169 | } 170 | 171 | // TODO: refactor writeMemory function to not need these 172 | event DidStep(bytes32 stateHash); 173 | function WriteMemoryWithReceipt(bytes32 stateHash, uint32 addr, uint32 value) public { 174 | bytes32 newStateHash = WriteMemory(stateHash, addr, value); 175 | emit DidStep(newStateHash); 176 | } 177 | 178 | function WriteBytes32WithReceipt(bytes32 stateHash, uint32 addr, bytes32 value) public { 179 | bytes32 newStateHash = WriteBytes32(stateHash, addr, value); 180 | emit DidStep(newStateHash); 181 | } 182 | 183 | // needed for preimage oracle 184 | function ReadBytes32(bytes32 stateHash, uint32 addr) public view returns (bytes32) { 185 | uint256 ret = 0; 186 | for (uint32 i = 0; i < 32; i += 4) { 187 | ret <<= 32; 188 | ret |= uint256(ReadMemory(stateHash, addr+i)); 189 | } 190 | return bytes32(ret); 191 | } 192 | 193 | function ReadMemory(bytes32 stateHash, uint32 addr) public view returns (uint32) { 194 | require(addr & 3 == 0, "read memory must be 32-bit aligned"); 195 | 196 | // zero register is always 0 197 | if (addr == 0xc0000000) { 198 | return 0; 199 | } 200 | 201 | // MMIO preimage oracle 202 | if (addr >= 0x31000000 && addr < 0x32000000) { 203 | bytes32 pihash = ReadBytes32(stateHash, 0x30001000); 204 | if (pihash == keccak256("")) { 205 | // both the length and any data are 0 206 | return 0; 207 | } 208 | if (addr == 0x31000000) { 209 | return uint32(GetPreimageLength(pihash)); 210 | } 211 | return GetPreimage(pihash, addr-0x31000004); 212 | } 213 | 214 | bool exists; 215 | bytes memory value; 216 | (exists, value) = Lib_MerkleTrie.get(tb(addr>>2), stateHash); 217 | 218 | if (!exists) { 219 | // this is uninitialized memory 220 | return 0; 221 | } else { 222 | return fb(value); 223 | } 224 | } 225 | 226 | // First read the size of data, the read the data 227 | function ReadMemoryToBytes(bytes32 stateHash, uint32 addr) public view returns (bytes memory) { 228 | require(addr & 3 == 0, "read memory must be 32-bit aligned"); 229 | uint32 size = ReadMemory(stateHash, addr); 230 | require(size & 3 == 0, "data must be 32-bit aligned"); 231 | bytes memory ret = new bytes(size); 232 | uint32 wordSize = size / 4; 233 | for (uint32 i = 0; i < wordSize; i += 4) { 234 | uint32 dat = ReadMemory(stateHash, addr+i); 235 | ret[i + 0] = bytes1(uint8(dat >> 24)); 236 | ret[i + 1] = bytes1(uint8(dat >> 16)); 237 | ret[i + 2] = bytes1(uint8(dat >> 8)); 238 | ret[i + 3] = bytes1(uint8(dat >> 0)); 239 | } 240 | return ret; 241 | } 242 | 243 | } -------------------------------------------------------------------------------- /contracts/lib/Lib_BytesUtils.sol: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | pragma solidity >0.5.0 <0.8.0; 3 | 4 | /** 5 | * @title Lib_BytesUtils 6 | */ 7 | library Lib_BytesUtils { 8 | 9 | /********************** 10 | * Internal Functions * 11 | **********************/ 12 | 13 | function concat( 14 | bytes memory _preBytes, 15 | bytes memory _postBytes 16 | ) 17 | internal 18 | pure 19 | returns (bytes memory) 20 | { 21 | bytes memory tempBytes; 22 | 23 | assembly { 24 | // Get a location of some free memory and store it in tempBytes as 25 | // Solidity does for memory variables. 26 | tempBytes := mload(0x40) 27 | 28 | // Store the length of the first bytes array at the beginning of 29 | // the memory for tempBytes. 30 | let length := mload(_preBytes) 31 | mstore(tempBytes, length) 32 | 33 | // Maintain a memory counter for the current write location in the 34 | // temp bytes array by adding the 32 bytes for the array length to 35 | // the starting location. 36 | let mc := add(tempBytes, 0x20) 37 | // Stop copying when the memory counter reaches the length of the 38 | // first bytes array. 39 | let end := add(mc, length) 40 | 41 | for { 42 | // Initialize a copy counter to the start of the _preBytes data, 43 | // 32 bytes into its memory. 44 | let cc := add(_preBytes, 0x20) 45 | } lt(mc, end) { 46 | // Increase both counters by 32 bytes each iteration. 47 | mc := add(mc, 0x20) 48 | cc := add(cc, 0x20) 49 | } { 50 | // Write the _preBytes data into the tempBytes memory 32 bytes 51 | // at a time. 52 | mstore(mc, mload(cc)) 53 | } 54 | 55 | // Add the length of _postBytes to the current length of tempBytes 56 | // and store it as the new length in the first 32 bytes of the 57 | // tempBytes memory. 58 | length := mload(_postBytes) 59 | mstore(tempBytes, add(length, mload(tempBytes))) 60 | 61 | // Move the memory counter back from a multiple of 0x20 to the 62 | // actual end of the _preBytes data. 63 | mc := end 64 | // Stop copying when the memory counter reaches the new combined 65 | // length of the arrays. 66 | end := add(mc, length) 67 | 68 | for { 69 | let cc := add(_postBytes, 0x20) 70 | } lt(mc, end) { 71 | mc := add(mc, 0x20) 72 | cc := add(cc, 0x20) 73 | } { 74 | mstore(mc, mload(cc)) 75 | } 76 | 77 | // Update the free-memory pointer by padding our last write location 78 | // to 32 bytes: add 31 bytes to the end of tempBytes to move to the 79 | // next 32 byte block, then round down to the nearest multiple of 80 | // 32. If the sum of the length of the two arrays is zero then add 81 | // one before rounding down to leave a blank 32 bytes (the length block with 0). 82 | mstore(0x40, and( 83 | add(add(end, iszero(add(length, mload(_preBytes)))), 31), 84 | not(31) // Round down to the nearest 32 bytes. 85 | )) 86 | } 87 | 88 | return tempBytes; 89 | } 90 | 91 | function slice( 92 | bytes memory _bytes, 93 | uint256 _start, 94 | uint256 _length 95 | ) 96 | internal 97 | pure 98 | returns (bytes memory) 99 | { 100 | require(_length + 31 >= _length, "slice_overflow"); 101 | require(_start + _length >= _start, "slice_overflow"); 102 | require(_bytes.length >= _start + _length, "slice_outOfBounds"); 103 | 104 | bytes memory tempBytes; 105 | 106 | assembly { 107 | switch iszero(_length) 108 | case 0 { 109 | // Get a location of some free memory and store it in tempBytes as 110 | // Solidity does for memory variables. 111 | tempBytes := mload(0x40) 112 | 113 | // The first word of the slice result is potentially a partial 114 | // word read from the original array. To read it, we calculate 115 | // the length of that partial word and start copying that many 116 | // bytes into the array. The first word we copy will start with 117 | // data we don't care about, but the last `lengthmod` bytes will 118 | // land at the beginning of the contents of the new array. When 119 | // we're done copying, we overwrite the full first word with 120 | // the actual length of the slice. 121 | let lengthmod := and(_length, 31) 122 | 123 | // The multiplication in the next line is necessary 124 | // because when slicing multiples of 32 bytes (lengthmod == 0) 125 | // the following copy loop was copying the origin's length 126 | // and then ending prematurely not copying everything it should. 127 | let mc := add(add(tempBytes, lengthmod), mul(0x20, iszero(lengthmod))) 128 | let end := add(mc, _length) 129 | 130 | for { 131 | // The multiplication in the next line has the same exact purpose 132 | // as the one above. 133 | let cc := add(add(add(_bytes, lengthmod), mul(0x20, iszero(lengthmod))), _start) 134 | } lt(mc, end) { 135 | mc := add(mc, 0x20) 136 | cc := add(cc, 0x20) 137 | } { 138 | mstore(mc, mload(cc)) 139 | } 140 | 141 | mstore(tempBytes, _length) 142 | 143 | //update free-memory pointer 144 | //allocating the array padded to 32 bytes like the compiler does now 145 | mstore(0x40, and(add(mc, 31), not(31))) 146 | } 147 | //if we want a zero-length slice let's just return a zero-length array 148 | default { 149 | tempBytes := mload(0x40) 150 | 151 | //zero out the 32 bytes slice we are about to return 152 | //we need to do it because Solidity does not garbage collect 153 | mstore(tempBytes, 0) 154 | 155 | mstore(0x40, add(tempBytes, 0x20)) 156 | } 157 | } 158 | 159 | return tempBytes; 160 | } 161 | 162 | function slice( 163 | bytes memory _bytes, 164 | uint256 _start 165 | ) 166 | internal 167 | pure 168 | returns (bytes memory) 169 | { 170 | if (_bytes.length - _start == 0) { 171 | return bytes(''); 172 | } 173 | 174 | return slice(_bytes, _start, _bytes.length - _start); 175 | } 176 | 177 | function toBytes32PadLeft( 178 | bytes memory _bytes 179 | ) 180 | internal 181 | pure 182 | returns (bytes32) 183 | { 184 | bytes32 ret; 185 | uint256 len = _bytes.length <= 32 ? _bytes.length : 32; 186 | assembly { 187 | ret := shr(mul(sub(32, len), 8), mload(add(_bytes, 32))) 188 | } 189 | return ret; 190 | } 191 | 192 | function toBytes32( 193 | bytes memory _bytes 194 | ) 195 | internal 196 | pure 197 | returns (bytes32) 198 | { 199 | if (_bytes.length < 32) { 200 | bytes32 ret; 201 | assembly { 202 | ret := mload(add(_bytes, 32)) 203 | } 204 | return ret; 205 | } 206 | 207 | return abi.decode(_bytes,(bytes32)); // will truncate if input length > 32 bytes 208 | } 209 | 210 | function toUint256( 211 | bytes memory _bytes 212 | ) 213 | internal 214 | pure 215 | returns (uint256) 216 | { 217 | return uint256(toBytes32(_bytes)); 218 | } 219 | 220 | function toUint24(bytes memory _bytes, uint256 _start) internal pure returns (uint24) { 221 | require(_start + 3 >= _start, "toUint24_overflow"); 222 | require(_bytes.length >= _start + 3 , "toUint24_outOfBounds"); 223 | uint24 tempUint; 224 | 225 | assembly { 226 | tempUint := mload(add(add(_bytes, 0x3), _start)) 227 | } 228 | 229 | return tempUint; 230 | } 231 | 232 | function toUint8(bytes memory _bytes, uint256 _start) internal pure returns (uint8) { 233 | require(_start + 1 >= _start, "toUint8_overflow"); 234 | require(_bytes.length >= _start + 1 , "toUint8_outOfBounds"); 235 | uint8 tempUint; 236 | 237 | assembly { 238 | tempUint := mload(add(add(_bytes, 0x1), _start)) 239 | } 240 | 241 | return tempUint; 242 | } 243 | 244 | function toAddress(bytes memory _bytes, uint256 _start) internal pure returns (address) { 245 | require(_start + 20 >= _start, "toAddress_overflow"); 246 | require(_bytes.length >= _start + 20, "toAddress_outOfBounds"); 247 | address tempAddress; 248 | 249 | assembly { 250 | tempAddress := div(mload(add(add(_bytes, 0x20), _start)), 0x1000000000000000000000000) 251 | } 252 | 253 | return tempAddress; 254 | } 255 | 256 | function revertWithHex( 257 | bytes memory _bytes 258 | ) 259 | internal 260 | pure 261 | { 262 | bytes memory node = Lib_BytesUtils.toNibbles(_bytes); 263 | for (uint i = 0; i < node.length; i++) { 264 | if (node[i] < bytes1(uint8(10))) { 265 | node[i] = bytes1(uint8(node[i]) + uint8(0x30)); 266 | } else { 267 | node[i] = bytes1(uint8(node[i]) + uint8(0x61-10)); 268 | } 269 | } 270 | revert(string(node)); 271 | } 272 | 273 | function toNibbles( 274 | bytes memory _bytes 275 | ) 276 | internal 277 | pure 278 | returns (bytes memory) 279 | { 280 | bytes memory nibbles = new bytes(_bytes.length * 2); 281 | 282 | for (uint256 i = 0; i < _bytes.length; i++) { 283 | nibbles[i * 2] = _bytes[i] >> 4; 284 | nibbles[i * 2 + 1] = bytes1(uint8(_bytes[i]) % 16); 285 | } 286 | 287 | return nibbles; 288 | } 289 | 290 | function fromNibbles( 291 | bytes memory _bytes 292 | ) 293 | internal 294 | pure 295 | returns (bytes memory) 296 | { 297 | bytes memory ret = new bytes(_bytes.length / 2); 298 | 299 | for (uint256 i = 0; i < ret.length; i++) { 300 | ret[i] = (_bytes[i * 2] << 4) | (_bytes[i * 2 + 1]); 301 | } 302 | 303 | return ret; 304 | } 305 | 306 | function equal( 307 | bytes memory _bytes, 308 | bytes memory _other 309 | ) 310 | internal 311 | pure 312 | returns (bool) 313 | { 314 | return keccak256(_bytes) == keccak256(_other); 315 | } 316 | } 317 | -------------------------------------------------------------------------------- /contracts/lib/Lib_Keccak256.sol: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | pragma solidity >0.5.0 <0.8.0; 3 | 4 | // https://chenglongma.com/10/simple-keccak/ 5 | // https://github.com/firefly/wallet/blob/master/source/libs/ethers/src/keccak256.c 6 | 7 | library Lib_Keccak256 { 8 | struct CTX { 9 | uint64[25] A; 10 | } 11 | 12 | function get_round_constant(uint round) internal pure returns (uint64) { 13 | uint64 result = 0; 14 | uint8 roundInfo = uint8(0x7421587966164852535d4f3f26350c0e5579211f705e1a01 >> (round*8)); 15 | result |= (uint64(roundInfo) << (63-6)) & (1 << 63); 16 | result |= (uint64(roundInfo) << (31-5)) & (1 << 31); 17 | result |= (uint64(roundInfo) << (15-4)) & (1 << 15); 18 | result |= (uint64(roundInfo) << (7-3)) & (1 << 7); 19 | result |= (uint64(roundInfo) << (3-2)) & (1 << 3); 20 | result |= (uint64(roundInfo) << (1-1)) & (1 << 1); 21 | result |= (uint64(roundInfo) << (0-0)) & (1 << 0); 22 | return result; 23 | } 24 | 25 | function keccak_theta_rho_pi(CTX memory c) internal pure { 26 | uint64 C0 = c.A[0] ^ c.A[5] ^ c.A[10] ^ c.A[15] ^ c.A[20]; 27 | uint64 C1 = c.A[1] ^ c.A[6] ^ c.A[11] ^ c.A[16] ^ c.A[21]; 28 | uint64 C2 = c.A[2] ^ c.A[7] ^ c.A[12] ^ c.A[17] ^ c.A[22]; 29 | uint64 C3 = c.A[3] ^ c.A[8] ^ c.A[13] ^ c.A[18] ^ c.A[23]; 30 | uint64 C4 = c.A[4] ^ c.A[9] ^ c.A[14] ^ c.A[19] ^ c.A[24]; 31 | uint64 D0 = (C1 << 1) ^ (C1 >> 63) ^ C4; 32 | uint64 D1 = (C2 << 1) ^ (C2 >> 63) ^ C0; 33 | uint64 D2 = (C3 << 1) ^ (C3 >> 63) ^ C1; 34 | uint64 D3 = (C4 << 1) ^ (C4 >> 63) ^ C2; 35 | uint64 D4 = (C0 << 1) ^ (C0 >> 63) ^ C3; 36 | c.A[0] ^= D0; 37 | uint64 A1 = ((c.A[1] ^ D1) << 1) ^ ((c.A[1] ^ D1) >> (64-1)); 38 | c.A[1] = ((c.A[6] ^ D1) << 44) ^ ((c.A[6] ^ D1) >> (64-44)); 39 | c.A[6] = ((c.A[9] ^ D4) << 20) ^ ((c.A[9] ^ D4) >> (64-20)); 40 | c.A[9] = ((c.A[22] ^ D2) << 61) ^ ((c.A[22] ^ D2) >> (64-61)); 41 | c.A[22] = ((c.A[14] ^ D4) << 39) ^ ((c.A[14] ^ D4) >> (64-39)); 42 | c.A[14] = ((c.A[20] ^ D0) << 18) ^ ((c.A[20] ^ D0) >> (64-18)); 43 | c.A[20] = ((c.A[2] ^ D2) << 62) ^ ((c.A[2] ^ D2) >> (64-62)); 44 | c.A[2] = ((c.A[12] ^ D2) << 43) ^ ((c.A[12] ^ D2) >> (64-43)); 45 | c.A[12] = ((c.A[13] ^ D3) << 25) ^ ((c.A[13] ^ D3) >> (64-25)); 46 | c.A[13] = ((c.A[19] ^ D4) << 8) ^ ((c.A[19] ^ D4) >> (64-8)); 47 | c.A[19] = ((c.A[23] ^ D3) << 56) ^ ((c.A[23] ^ D3) >> (64-56)); 48 | c.A[23] = ((c.A[15] ^ D0) << 41) ^ ((c.A[15] ^ D0) >> (64-41)); 49 | c.A[15] = ((c.A[4] ^ D4) << 27) ^ ((c.A[4] ^ D4) >> (64-27)); 50 | c.A[4] = ((c.A[24] ^ D4) << 14) ^ ((c.A[24] ^ D4) >> (64-14)); 51 | c.A[24] = ((c.A[21] ^ D1) << 2) ^ ((c.A[21] ^ D1) >> (64-2)); 52 | c.A[21] = ((c.A[8] ^ D3) << 55) ^ ((c.A[8] ^ D3) >> (64-55)); 53 | c.A[8] = ((c.A[16] ^ D1) << 45) ^ ((c.A[16] ^ D1) >> (64-45)); 54 | c.A[16] = ((c.A[5] ^ D0) << 36) ^ ((c.A[5] ^ D0) >> (64-36)); 55 | c.A[5] = ((c.A[3] ^ D3) << 28) ^ ((c.A[3] ^ D3) >> (64-28)); 56 | c.A[3] = ((c.A[18] ^ D3) << 21) ^ ((c.A[18] ^ D3) >> (64-21)); 57 | c.A[18] = ((c.A[17] ^ D2) << 15) ^ ((c.A[17] ^ D2) >> (64-15)); 58 | c.A[17] = ((c.A[11] ^ D1) << 10) ^ ((c.A[11] ^ D1) >> (64-10)); 59 | c.A[11] = ((c.A[7] ^ D2) << 6) ^ ((c.A[7] ^ D2) >> (64-6)); 60 | c.A[7] = ((c.A[10] ^ D0) << 3) ^ ((c.A[10] ^ D0) >> (64-3)); 61 | c.A[10] = A1; 62 | } 63 | 64 | function keccak_chi(CTX memory c) internal pure { 65 | uint i; 66 | uint64 A0; 67 | uint64 A1; 68 | uint64 A2; 69 | uint64 A3; 70 | uint64 A4; 71 | for (i = 0; i < 25; i+=5) { 72 | A0 = c.A[0 + i]; 73 | A1 = c.A[1 + i]; 74 | A2 = c.A[2 + i]; 75 | A3 = c.A[3 + i]; 76 | A4 = c.A[4 + i]; 77 | c.A[0 + i] ^= ~A1 & A2; 78 | c.A[1 + i] ^= ~A2 & A3; 79 | c.A[2 + i] ^= ~A3 & A4; 80 | c.A[3 + i] ^= ~A4 & A0; 81 | c.A[4 + i] ^= ~A0 & A1; 82 | } 83 | } 84 | 85 | function keccak_init(CTX memory c) internal pure { 86 | // is this needed? 87 | uint i; 88 | for (i = 0; i < 25; i++) { 89 | c.A[i] = 0; 90 | } 91 | } 92 | 93 | function sha3_xor_input(CTX memory c, bytes memory dat) internal pure { 94 | for (uint i = 0; i < 17; i++) { 95 | uint bo = i*8; 96 | c.A[i] ^= uint64(uint8(dat[bo+7])) << 56 | 97 | uint64(uint8(dat[bo+6])) << 48 | 98 | uint64(uint8(dat[bo+5])) << 40 | 99 | uint64(uint8(dat[bo+4])) << 32 | 100 | uint64(uint8(dat[bo+3])) << 24 | 101 | uint64(uint8(dat[bo+2])) << 16 | 102 | uint64(uint8(dat[bo+1])) << 8 | 103 | uint64(uint8(dat[bo+0])) << 0; 104 | } 105 | } 106 | 107 | function sha3_permutation(CTX memory c) internal pure { 108 | uint round; 109 | for (round = 0; round < 24; round++) { 110 | keccak_theta_rho_pi(c); 111 | keccak_chi(c); 112 | // keccak_iota 113 | c.A[0] ^= get_round_constant(round); 114 | } 115 | } 116 | 117 | // https://stackoverflow.com/questions/2182002/convert-big-endian-to-little-endian-in-c-without-using-provided-func 118 | function flip(uint64 val) internal pure returns (uint64) { 119 | val = ((val << 8) & 0xFF00FF00FF00FF00 ) | ((val >> 8) & 0x00FF00FF00FF00FF ); 120 | val = ((val << 16) & 0xFFFF0000FFFF0000 ) | ((val >> 16) & 0x0000FFFF0000FFFF ); 121 | return (val << 32) | (val >> 32); 122 | } 123 | 124 | function get_hash(CTX memory c) internal pure returns (bytes32) { 125 | return bytes32((uint256(flip(c.A[0])) << 192) | 126 | (uint256(flip(c.A[1])) << 128) | 127 | (uint256(flip(c.A[2])) << 64) | 128 | (uint256(flip(c.A[3])) << 0)); 129 | } 130 | 131 | } -------------------------------------------------------------------------------- /contracts/lib/Lib_RLPReader.sol: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | pragma solidity >0.5.0 <0.8.0; 3 | 4 | /** 5 | * @title Lib_RLPReader 6 | * @dev Adapted from "RLPReader" by Hamdi Allam (hamdi.allam97@gmail.com). 7 | */ 8 | library Lib_RLPReader { 9 | 10 | /************* 11 | * Constants * 12 | *************/ 13 | 14 | uint256 constant internal MAX_LIST_LENGTH = 32; 15 | 16 | 17 | /********* 18 | * Enums * 19 | *********/ 20 | 21 | enum RLPItemType { 22 | DATA_ITEM, 23 | LIST_ITEM 24 | } 25 | 26 | 27 | /*********** 28 | * Structs * 29 | ***********/ 30 | 31 | struct RLPItem { 32 | uint256 length; 33 | uint256 ptr; 34 | } 35 | 36 | 37 | /********************** 38 | * Internal Functions * 39 | **********************/ 40 | 41 | /** 42 | * Converts bytes to a reference to memory position and length. 43 | * @param _in Input bytes to convert. 44 | * @return Output memory reference. 45 | */ 46 | function toRLPItem( 47 | bytes memory _in 48 | ) 49 | internal 50 | pure 51 | returns ( 52 | RLPItem memory 53 | ) 54 | { 55 | uint256 ptr; 56 | assembly { 57 | ptr := add(_in, 32) 58 | } 59 | 60 | return RLPItem({ 61 | length: _in.length, 62 | ptr: ptr 63 | }); 64 | } 65 | 66 | /** 67 | * Reads an RLP list value into a list of RLP items. 68 | * @param _in RLP list value. 69 | * @return Decoded RLP list items. 70 | */ 71 | function readList( 72 | RLPItem memory _in 73 | ) 74 | internal 75 | pure 76 | returns ( 77 | RLPItem[] memory 78 | ) 79 | { 80 | ( 81 | uint256 listOffset, 82 | , 83 | RLPItemType itemType 84 | ) = _decodeLength(_in); 85 | 86 | require( 87 | itemType == RLPItemType.LIST_ITEM, 88 | "Invalid RLP list value." 89 | ); 90 | 91 | // Solidity in-memory arrays can't be increased in size, but *can* be decreased in size by 92 | // writing to the length. Since we can't know the number of RLP items without looping over 93 | // the entire input, we'd have to loop twice to accurately size this array. It's easier to 94 | // simply set a reasonable maximum list length and decrease the size before we finish. 95 | RLPItem[] memory out = new RLPItem[](MAX_LIST_LENGTH); 96 | 97 | uint256 itemCount = 0; 98 | uint256 offset = listOffset; 99 | while (offset < _in.length) { 100 | require( 101 | itemCount < MAX_LIST_LENGTH, 102 | "Provided RLP list exceeds max list length." 103 | ); 104 | 105 | ( 106 | uint256 itemOffset, 107 | uint256 itemLength, 108 | ) = _decodeLength(RLPItem({ 109 | length: _in.length - offset, 110 | ptr: _in.ptr + offset 111 | })); 112 | 113 | out[itemCount] = RLPItem({ 114 | length: itemLength + itemOffset, 115 | ptr: _in.ptr + offset 116 | }); 117 | 118 | itemCount += 1; 119 | offset += itemOffset + itemLength; 120 | } 121 | 122 | // Decrease the array size to match the actual item count. 123 | assembly { 124 | mstore(out, itemCount) 125 | } 126 | 127 | return out; 128 | } 129 | 130 | /** 131 | * Reads an RLP list value into a list of RLP items. 132 | * @param _in RLP list value. 133 | * @return Decoded RLP list items. 134 | */ 135 | function readList( 136 | bytes memory _in 137 | ) 138 | internal 139 | pure 140 | returns ( 141 | RLPItem[] memory 142 | ) 143 | { 144 | return readList( 145 | toRLPItem(_in) 146 | ); 147 | } 148 | 149 | /** 150 | * Reads an RLP bytes value into bytes. 151 | * @param _in RLP bytes value. 152 | * @return Decoded bytes. 153 | */ 154 | function readBytes( 155 | RLPItem memory _in 156 | ) 157 | internal 158 | pure 159 | returns ( 160 | bytes memory 161 | ) 162 | { 163 | ( 164 | uint256 itemOffset, 165 | uint256 itemLength, 166 | RLPItemType itemType 167 | ) = _decodeLength(_in); 168 | 169 | require( 170 | itemType == RLPItemType.DATA_ITEM, 171 | "Invalid RLP bytes value." 172 | ); 173 | 174 | return _copy(_in.ptr, itemOffset, itemLength); 175 | } 176 | 177 | /** 178 | * Reads an RLP bytes value into bytes. 179 | * @param _in RLP bytes value. 180 | * @return Decoded bytes. 181 | */ 182 | function readBytes( 183 | bytes memory _in 184 | ) 185 | internal 186 | pure 187 | returns ( 188 | bytes memory 189 | ) 190 | { 191 | return readBytes( 192 | toRLPItem(_in) 193 | ); 194 | } 195 | 196 | /** 197 | * Reads an RLP string value into a string. 198 | * @param _in RLP string value. 199 | * @return Decoded string. 200 | */ 201 | function readString( 202 | RLPItem memory _in 203 | ) 204 | internal 205 | pure 206 | returns ( 207 | string memory 208 | ) 209 | { 210 | return string(readBytes(_in)); 211 | } 212 | 213 | /** 214 | * Reads an RLP string value into a string. 215 | * @param _in RLP string value. 216 | * @return Decoded string. 217 | */ 218 | function readString( 219 | bytes memory _in 220 | ) 221 | internal 222 | pure 223 | returns ( 224 | string memory 225 | ) 226 | { 227 | return readString( 228 | toRLPItem(_in) 229 | ); 230 | } 231 | 232 | /** 233 | * Reads an RLP bytes32 value into a bytes32. 234 | * @param _in RLP bytes32 value. 235 | * @return Decoded bytes32. 236 | */ 237 | function readBytes32( 238 | RLPItem memory _in 239 | ) 240 | internal 241 | pure 242 | returns ( 243 | bytes32 244 | ) 245 | { 246 | require( 247 | _in.length <= 33, 248 | "Invalid RLP bytes32 value." 249 | ); 250 | 251 | ( 252 | uint256 itemOffset, 253 | uint256 itemLength, 254 | RLPItemType itemType 255 | ) = _decodeLength(_in); 256 | 257 | require( 258 | itemType == RLPItemType.DATA_ITEM, 259 | "Invalid RLP bytes32 value." 260 | ); 261 | 262 | uint256 ptr = _in.ptr + itemOffset; 263 | bytes32 out; 264 | assembly { 265 | out := mload(ptr) 266 | 267 | // Shift the bytes over to match the item size. 268 | if lt(itemLength, 32) { 269 | out := div(out, exp(256, sub(32, itemLength))) 270 | } 271 | } 272 | 273 | return out; 274 | } 275 | 276 | /** 277 | * Reads an RLP bytes32 value into a bytes32. 278 | * @param _in RLP bytes32 value. 279 | * @return Decoded bytes32. 280 | */ 281 | function readBytes32( 282 | bytes memory _in 283 | ) 284 | internal 285 | pure 286 | returns ( 287 | bytes32 288 | ) 289 | { 290 | return readBytes32( 291 | toRLPItem(_in) 292 | ); 293 | } 294 | 295 | /** 296 | * Reads an RLP uint256 value into a uint256. 297 | * @param _in RLP uint256 value. 298 | * @return Decoded uint256. 299 | */ 300 | function readUint256( 301 | RLPItem memory _in 302 | ) 303 | internal 304 | pure 305 | returns ( 306 | uint256 307 | ) 308 | { 309 | return uint256(readBytes32(_in)); 310 | } 311 | 312 | /** 313 | * Reads an RLP uint256 value into a uint256. 314 | * @param _in RLP uint256 value. 315 | * @return Decoded uint256. 316 | */ 317 | function readUint256( 318 | bytes memory _in 319 | ) 320 | internal 321 | pure 322 | returns ( 323 | uint256 324 | ) 325 | { 326 | return readUint256( 327 | toRLPItem(_in) 328 | ); 329 | } 330 | 331 | /** 332 | * Reads an RLP bool value into a bool. 333 | * @param _in RLP bool value. 334 | * @return Decoded bool. 335 | */ 336 | function readBool( 337 | RLPItem memory _in 338 | ) 339 | internal 340 | pure 341 | returns ( 342 | bool 343 | ) 344 | { 345 | require( 346 | _in.length == 1, 347 | "Invalid RLP boolean value." 348 | ); 349 | 350 | uint256 ptr = _in.ptr; 351 | uint256 out; 352 | assembly { 353 | out := byte(0, mload(ptr)) 354 | } 355 | 356 | require( 357 | out == 0 || out == 1, 358 | "Lib_RLPReader: Invalid RLP boolean value, must be 0 or 1" 359 | ); 360 | 361 | return out != 0; 362 | } 363 | 364 | /** 365 | * Reads an RLP bool value into a bool. 366 | * @param _in RLP bool value. 367 | * @return Decoded bool. 368 | */ 369 | function readBool( 370 | bytes memory _in 371 | ) 372 | internal 373 | pure 374 | returns ( 375 | bool 376 | ) 377 | { 378 | return readBool( 379 | toRLPItem(_in) 380 | ); 381 | } 382 | 383 | /** 384 | * Reads an RLP address value into a address. 385 | * @param _in RLP address value. 386 | * @return Decoded address. 387 | */ 388 | function readAddress( 389 | RLPItem memory _in 390 | ) 391 | internal 392 | pure 393 | returns ( 394 | address 395 | ) 396 | { 397 | if (_in.length == 1) { 398 | return address(0); 399 | } 400 | 401 | require( 402 | _in.length == 21, 403 | "Invalid RLP address value." 404 | ); 405 | 406 | return address(readUint256(_in)); 407 | } 408 | 409 | /** 410 | * Reads an RLP address value into a address. 411 | * @param _in RLP address value. 412 | * @return Decoded address. 413 | */ 414 | function readAddress( 415 | bytes memory _in 416 | ) 417 | internal 418 | pure 419 | returns ( 420 | address 421 | ) 422 | { 423 | return readAddress( 424 | toRLPItem(_in) 425 | ); 426 | } 427 | 428 | /** 429 | * Reads the raw bytes of an RLP item. 430 | * @param _in RLP item to read. 431 | * @return Raw RLP bytes. 432 | */ 433 | function readRawBytes( 434 | RLPItem memory _in 435 | ) 436 | internal 437 | pure 438 | returns ( 439 | bytes memory 440 | ) 441 | { 442 | return _copy(_in); 443 | } 444 | 445 | 446 | /********************* 447 | * Private Functions * 448 | *********************/ 449 | 450 | /** 451 | * Decodes the length of an RLP item. 452 | * @param _in RLP item to decode. 453 | * @return Offset of the encoded data. 454 | * @return Length of the encoded data. 455 | * @return RLP item type (LIST_ITEM or DATA_ITEM). 456 | */ 457 | function _decodeLength( 458 | RLPItem memory _in 459 | ) 460 | internal 461 | pure 462 | returns ( 463 | uint256, 464 | uint256, 465 | RLPItemType 466 | ) 467 | { 468 | require( 469 | _in.length > 0, 470 | "RLP item cannot be null." 471 | ); 472 | 473 | uint256 ptr = _in.ptr; 474 | uint256 prefix; 475 | assembly { 476 | prefix := byte(0, mload(ptr)) 477 | } 478 | 479 | if (prefix <= 0x7f) { 480 | // Single byte. 481 | 482 | return (0, 1, RLPItemType.DATA_ITEM); 483 | } else if (prefix <= 0xb7) { 484 | // Short string. 485 | 486 | uint256 strLen = prefix - 0x80; 487 | 488 | require( 489 | _in.length > strLen, 490 | "Invalid RLP short string." 491 | ); 492 | 493 | return (1, strLen, RLPItemType.DATA_ITEM); 494 | } else if (prefix <= 0xbf) { 495 | // Long string. 496 | uint256 lenOfStrLen = prefix - 0xb7; 497 | 498 | require( 499 | _in.length > lenOfStrLen, 500 | "Invalid RLP long string length." 501 | ); 502 | 503 | uint256 strLen; 504 | assembly { 505 | // Pick out the string length. 506 | strLen := div( 507 | mload(add(ptr, 1)), 508 | exp(256, sub(32, lenOfStrLen)) 509 | ) 510 | } 511 | 512 | require( 513 | _in.length > lenOfStrLen + strLen, 514 | "Invalid RLP long string." 515 | ); 516 | 517 | return (1 + lenOfStrLen, strLen, RLPItemType.DATA_ITEM); 518 | } else if (prefix <= 0xf7) { 519 | // Short list. 520 | uint256 listLen = prefix - 0xc0; 521 | 522 | require( 523 | _in.length > listLen, 524 | "Invalid RLP short list." 525 | ); 526 | 527 | return (1, listLen, RLPItemType.LIST_ITEM); 528 | } else { 529 | // Long list. 530 | uint256 lenOfListLen = prefix - 0xf7; 531 | 532 | require( 533 | _in.length > lenOfListLen, 534 | "Invalid RLP long list length." 535 | ); 536 | 537 | uint256 listLen; 538 | assembly { 539 | // Pick out the list length. 540 | listLen := div( 541 | mload(add(ptr, 1)), 542 | exp(256, sub(32, lenOfListLen)) 543 | ) 544 | } 545 | 546 | require( 547 | _in.length > lenOfListLen + listLen, 548 | "Invalid RLP long list." 549 | ); 550 | 551 | return (1 + lenOfListLen, listLen, RLPItemType.LIST_ITEM); 552 | } 553 | } 554 | 555 | /** 556 | * Copies the bytes from a memory location. 557 | * @param _src Pointer to the location to read from. 558 | * @param _offset Offset to start reading from. 559 | * @param _length Number of bytes to read. 560 | * @return Copied bytes. 561 | */ 562 | function _copy( 563 | uint256 _src, 564 | uint256 _offset, 565 | uint256 _length 566 | ) 567 | internal 568 | pure 569 | returns ( 570 | bytes memory 571 | ) 572 | { 573 | bytes memory out = new bytes(_length); 574 | if (out.length == 0) { 575 | return out; 576 | } 577 | 578 | uint256 src = _src + _offset; 579 | uint256 dest; 580 | assembly { 581 | dest := add(out, 32) 582 | } 583 | 584 | // Copy over as many complete words as we can. 585 | for (uint256 i = 0; i < _length / 32; i++) { 586 | assembly { 587 | mstore(dest, mload(src)) 588 | } 589 | 590 | src += 32; 591 | dest += 32; 592 | } 593 | 594 | // Pick out the remaining bytes. 595 | uint256 mask = 256 ** (32 - (_length % 32)) - 1; 596 | assembly { 597 | mstore( 598 | dest, 599 | or( 600 | and(mload(src), not(mask)), 601 | and(mload(dest), mask) 602 | ) 603 | ) 604 | } 605 | 606 | return out; 607 | } 608 | 609 | /** 610 | * Copies an RLP item into bytes. 611 | * @param _in RLP item to copy. 612 | * @return Copied bytes. 613 | */ 614 | function _copy( 615 | RLPItem memory _in 616 | ) 617 | private 618 | pure 619 | returns ( 620 | bytes memory 621 | ) 622 | { 623 | return _copy(_in.ptr, 0, _in.length); 624 | } 625 | } 626 | -------------------------------------------------------------------------------- /contracts/lib/Lib_RLPWriter.sol: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | pragma solidity >0.5.0 <0.8.0; 3 | pragma experimental ABIEncoderV2; 4 | 5 | /** 6 | * @title Lib_RLPWriter 7 | * @author Bakaoh (with modifications) 8 | */ 9 | library Lib_RLPWriter { 10 | 11 | /********************** 12 | * Internal Functions * 13 | **********************/ 14 | 15 | /** 16 | * RLP encodes a byte string. 17 | * @param _in The byte string to encode. 18 | * @return The RLP encoded string in bytes. 19 | */ 20 | function writeBytes( 21 | bytes memory _in 22 | ) 23 | internal 24 | pure 25 | returns ( 26 | bytes memory 27 | ) 28 | { 29 | bytes memory encoded; 30 | 31 | if (_in.length == 1 && uint8(_in[0]) < 128) { 32 | encoded = _in; 33 | } else { 34 | encoded = abi.encodePacked(_writeLength(_in.length, 128), _in); 35 | } 36 | 37 | return encoded; 38 | } 39 | 40 | /** 41 | * RLP encodes a list of RLP encoded byte byte strings. 42 | * @param _in The list of RLP encoded byte strings. 43 | * @return The RLP encoded list of items in bytes. 44 | */ 45 | function writeList( 46 | bytes[] memory _in 47 | ) 48 | internal 49 | pure 50 | returns ( 51 | bytes memory 52 | ) 53 | { 54 | bytes memory list = _flatten(_in); 55 | return abi.encodePacked(_writeLength(list.length, 192), list); 56 | } 57 | 58 | /** 59 | * RLP encodes a string. 60 | * @param _in The string to encode. 61 | * @return The RLP encoded string in bytes. 62 | */ 63 | function writeString( 64 | string memory _in 65 | ) 66 | internal 67 | pure 68 | returns ( 69 | bytes memory 70 | ) 71 | { 72 | return writeBytes(bytes(_in)); 73 | } 74 | 75 | /** 76 | * RLP encodes an address. 77 | * @param _in The address to encode. 78 | * @return The RLP encoded address in bytes. 79 | */ 80 | function writeAddress( 81 | address _in 82 | ) 83 | internal 84 | pure 85 | returns ( 86 | bytes memory 87 | ) 88 | { 89 | return writeBytes(abi.encodePacked(_in)); 90 | } 91 | 92 | /** 93 | * RLP encodes a bytes32 value. 94 | * @param _in The bytes32 to encode. 95 | * @return _out The RLP encoded bytes32 in bytes. 96 | */ 97 | function writeBytes32( 98 | bytes32 _in 99 | ) 100 | internal 101 | pure 102 | returns ( 103 | bytes memory _out 104 | ) 105 | { 106 | return writeBytes(abi.encodePacked(_in)); 107 | } 108 | 109 | /** 110 | * RLP encodes a uint. 111 | * @param _in The uint256 to encode. 112 | * @return The RLP encoded uint256 in bytes. 113 | */ 114 | function writeUint( 115 | uint256 _in 116 | ) 117 | internal 118 | pure 119 | returns ( 120 | bytes memory 121 | ) 122 | { 123 | return writeBytes(_toBinary(_in)); 124 | } 125 | 126 | /** 127 | * RLP encodes a bool. 128 | * @param _in The bool to encode. 129 | * @return The RLP encoded bool in bytes. 130 | */ 131 | function writeBool( 132 | bool _in 133 | ) 134 | internal 135 | pure 136 | returns ( 137 | bytes memory 138 | ) 139 | { 140 | bytes memory encoded = new bytes(1); 141 | encoded[0] = (_in ? bytes1(0x01) : bytes1(0x80)); 142 | return encoded; 143 | } 144 | 145 | 146 | /********************* 147 | * Private Functions * 148 | *********************/ 149 | 150 | /** 151 | * Encode the first byte, followed by the `len` in binary form if `length` is more than 55. 152 | * @param _len The length of the string or the payload. 153 | * @param _offset 128 if item is string, 192 if item is list. 154 | * @return RLP encoded bytes. 155 | */ 156 | function _writeLength( 157 | uint256 _len, 158 | uint256 _offset 159 | ) 160 | private 161 | pure 162 | returns ( 163 | bytes memory 164 | ) 165 | { 166 | bytes memory encoded; 167 | 168 | if (_len < 56) { 169 | encoded = new bytes(1); 170 | encoded[0] = byte(uint8(_len) + uint8(_offset)); 171 | } else { 172 | uint256 lenLen; 173 | uint256 i = 1; 174 | while (_len / i != 0) { 175 | lenLen++; 176 | i *= 256; 177 | } 178 | 179 | encoded = new bytes(lenLen + 1); 180 | encoded[0] = byte(uint8(lenLen) + uint8(_offset) + 55); 181 | for(i = 1; i <= lenLen; i++) { 182 | encoded[i] = byte(uint8((_len / (256**(lenLen-i))) % 256)); 183 | } 184 | } 185 | 186 | return encoded; 187 | } 188 | 189 | /** 190 | * Encode integer in big endian binary form with no leading zeroes. 191 | * @notice TODO: This should be optimized with assembly to save gas costs. 192 | * @param _x The integer to encode. 193 | * @return RLP encoded bytes. 194 | */ 195 | function _toBinary( 196 | uint256 _x 197 | ) 198 | private 199 | pure 200 | returns ( 201 | bytes memory 202 | ) 203 | { 204 | bytes memory b = abi.encodePacked(_x); 205 | 206 | uint256 i = 0; 207 | for (; i < 32; i++) { 208 | if (b[i] != 0) { 209 | break; 210 | } 211 | } 212 | 213 | bytes memory res = new bytes(32 - i); 214 | for (uint256 j = 0; j < res.length; j++) { 215 | res[j] = b[i++]; 216 | } 217 | 218 | return res; 219 | } 220 | 221 | /** 222 | * Copies a piece of memory to another location. 223 | * @notice From: https://github.com/Arachnid/solidity-stringutils/blob/master/src/strings.sol. 224 | * @param _dest Destination location. 225 | * @param _src Source location. 226 | * @param _len Length of memory to copy. 227 | */ 228 | function _memcpy( 229 | uint256 _dest, 230 | uint256 _src, 231 | uint256 _len 232 | ) 233 | private 234 | pure 235 | { 236 | uint256 dest = _dest; 237 | uint256 src = _src; 238 | uint256 len = _len; 239 | 240 | for(; len >= 32; len -= 32) { 241 | assembly { 242 | mstore(dest, mload(src)) 243 | } 244 | dest += 32; 245 | src += 32; 246 | } 247 | 248 | uint256 mask = 256 ** (32 - len) - 1; 249 | assembly { 250 | let srcpart := and(mload(src), not(mask)) 251 | let destpart := and(mload(dest), mask) 252 | mstore(dest, or(destpart, srcpart)) 253 | } 254 | } 255 | 256 | /** 257 | * Flattens a list of byte strings into one byte string. 258 | * @notice From: https://github.com/sammayo/solidity-rlp-encoder/blob/master/RLPEncode.sol. 259 | * @param _list List of byte strings to flatten. 260 | * @return The flattened byte string. 261 | */ 262 | function _flatten( 263 | bytes[] memory _list 264 | ) 265 | private 266 | pure 267 | returns ( 268 | bytes memory 269 | ) 270 | { 271 | if (_list.length == 0) { 272 | return new bytes(0); 273 | } 274 | 275 | uint256 len; 276 | uint256 i = 0; 277 | for (; i < _list.length; i++) { 278 | len += _list[i].length; 279 | } 280 | 281 | bytes memory flattened = new bytes(len); 282 | uint256 flattenedPtr; 283 | assembly { flattenedPtr := add(flattened, 0x20) } 284 | 285 | for(i = 0; i < _list.length; i++) { 286 | bytes memory item = _list[i]; 287 | 288 | uint256 listPtr; 289 | assembly { listPtr := add(item, 0x20)} 290 | 291 | _memcpy(flattenedPtr, listPtr, item.length); 292 | flattenedPtr += _list[i].length; 293 | } 294 | 295 | return flattened; 296 | } 297 | } 298 | -------------------------------------------------------------------------------- /demo/challenge_simple.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # The following variables can be overridden as environment variables: 4 | # * BLOCK (block whose transition will be challenged) 5 | # * WRONG_BLOCK (block number used by challenger) 6 | # * SKIP_NODE (skip forking a node, useful if you've already forked a node) 7 | # 8 | # Example usage: 9 | # SKIP_NODE=1 BLOCK=13284469 WRONG_BLOCK=13284491 ./demo/challenge_simple.sh 10 | 11 | # --- DOC ---------------------------------------------------------------------- 12 | 13 | # In this example, the challenger will challenge the transition from a block 14 | # (`BLOCK`), but pretends that chain state before another block (`WRONG_BLOCK`) 15 | # is the state before the challenged block. Consequently, the challenger will 16 | # disagree with the defender on every single step of the challenge game, and the 17 | # single step to execute will be the very first MIPS instruction executed. The 18 | # reason is that the initial MIPS state Merkle root is stored on-chain, and 19 | # immediately modified to reflect the fact that the input hash for the block is 20 | # written at address 0x3000000. 21 | # 22 | # (The input hash is automatically validated against the blockhash, so note that 23 | # in this demo the challenger has to provide the correct (`BLOCK`) input hash to 24 | # the `initiateChallenge` function of `Challenge.sol`, but will execute as 25 | # though the input hash was the one derived from `WRONG_BLOCK`.) 26 | # 27 | # Because the challenger uses the wrong inputs, it will assert a post-state 28 | # (Merkle root) for the first MIPS instruction that has the wrong input hash at 29 | # 0x3000000. Hence, the challenge will fail. 30 | 31 | 32 | # --- SCRIPT SETUP ------------------------------------------------------------- 33 | 34 | shout() { 35 | echo "" 36 | echo "----------------------------------------" 37 | echo "$1" 38 | echo "----------------------------------------" 39 | echo "" 40 | } 41 | 42 | # Exit if any command fails. 43 | set -e 44 | 45 | exit_trap() { 46 | # Print an error if the last command failed 47 | # (in which case the script is exiting because of set -e). 48 | [[ $? == 0 ]] && return 49 | echo "----------------------------------------" 50 | echo "EARLY EXIT: SCRIPT FAILED" 51 | echo "----------------------------------------" 52 | 53 | # Kill (send SIGTERM) to the whole process group, also killing 54 | # any background processes. 55 | # I think the trap command resets SIGTERM before resending it to the whole 56 | # group. (cf. https://stackoverflow.com/a/2173421) 57 | trap - SIGTERM && kill -- -$$ 58 | } 59 | trap "exit_trap" SIGINT SIGTERM EXIT 60 | 61 | 62 | 63 | # --- CHALLENGE SETUP ---------------------------------------------------------- 64 | 65 | # AI model (mnist model) 66 | PROGRAM_PATH="./mlgo/examples/mnist_mips/mlgo.bin" 67 | MODEL_PATH="./mlgo/examples/mnist/models/mnist/ggml-model-small-f32-big-endian.bin" 68 | DATA_PATH="./mlgo/examples/mnist/models/mnist/input_7" 69 | 70 | export PROGRAM_PATH=$PROGRAM_PATH 71 | export MODEL_PATH=$MODEL_PATH 72 | export DATA_PATH=$DATA_PATH 73 | 74 | # challenge ID, read by respond.js and assert.js 75 | export ID=0 76 | 77 | # clear data from previous runs 78 | rm -rf /tmp/cannon/* /tmp/cannon_fault/* 79 | mkdir -p /tmp/cannon 80 | mkdir -p /tmp/cannon_fault 81 | 82 | # stored in /tmp/cannon/golden.json 83 | shout "GENERATING INITIAL MEMORY STATE CHECKPOINT" 84 | mlvm/mlvm --outputGolden --basedir=/tmp/cannon --program="$PROGRAM_PATH" --model="$MODEL_PATH" --data="$DATA_PATH" --mipsVMCompatible 85 | 86 | shout "DEPLOYING CONTRACTS" 87 | npx hardhat run scripts/deploy.js --network localhost 88 | 89 | # challenger will use same initial memory checkpoint and deployed contracts 90 | cp /tmp/cannon/{golden,deployed}.json /tmp/cannon_fault/ 91 | 92 | shout "COMPUTING FAKE MIPS FINAL MEMORY CHECKPOINT" 93 | BASEDIR=/tmp/cannon_fault mlvm/mlvm --program="$PROGRAM_PATH" --model="$MODEL_PATH" --data="$DATA_PATH" --mipsVMCompatible 94 | 95 | 96 | # --- BINARY SEARCH ------------------------------------------------------------ 97 | 98 | shout "STARTING CHALLENGE" 99 | BASEDIR=/tmp/cannon_fault npx hardhat run scripts/challenge.js --network localhost 100 | 101 | shout "BINARY SEARCH" 102 | for i in {1..25}; do 103 | echo "" 104 | echo "--- STEP $i / 25 ---" 105 | echo "" 106 | BASEDIR=/tmp/cannon_fault CHALLENGER=1 npx hardhat run scripts/respond.js --network localhost 107 | BASEDIR=/tmp/cannon CHALLENGER=0 npx hardhat run scripts/respond.js --network localhost 108 | done 109 | 110 | # --- SINGLE STEP EXECUTION ---------------------------------------------------- 111 | 112 | # shout "ASSERTING AS CHALLENGER (should fail)" 113 | # set +e # this should fail! 114 | # BASEDIR=/tmp/cannon_fault CHALLENGER=1 npx hardhat run scripts/assert.js --network localhost 115 | # set -e 116 | 117 | shout "ASSERTING AS DEFENDER (should pass)" 118 | npx hardhat run scripts/assert.js --network localhost 119 | -------------------------------------------------------------------------------- /docs/OPML.md: -------------------------------------------------------------------------------- 1 | # OPML: Optimistic Machine Learning on Blockchain 2 | 3 | ## TL;DR 4 | 5 | - We propose OPML (Optimistic Machine Learning), which enables AI model inference and training/fine-tuning on the blockchain system using optimistic approach. 6 | - OPML can provide ML service with low cost and high efficiency compared to ZKML. The participation requirement for OPML is low: We are now able to run OPML with a large language model, e.g., 7B-LLaMA (the model size is around 26GB) on a common PC without GPU. 7 | - OPML adopts a verification game (similar to Truebit and Optimistic Rollup systems) to guarantee decentralized and verifiable consensus on the ML service. 8 | - The requester first initiates an ML service task. 9 | - The server then finishes the ML service task and commits results on chain. 10 | - The verifier will validate the results. Suppose there exists a verifier who declares the results are wrong. It starts a verification game with verification game (bisection protocol) with the server and tries to disprove the claim by pinpointing one concrete erroneous step. 11 | - Finally, arbitration about a single step will be conducted on smart contract. 12 | 13 | ## Single-Phase Verification Game 14 | 15 | The one-phase pinpoint protocol works similarly to referred delegation of computation (RDoC), where two or more parties (with at least one honest party) are assumed to execute the same program. Then, the parties can challenge each other with a pinpoint style to locate the disputable step. The step is sent to a judge with weak computation power (smart contract on blockchain) for arbitration. 16 | 17 | In one-phase OPML: 18 | 19 | - We build a virtual machine (VM) for off-chain execution and on-chain arbitration. We guarantee the equivalence of the off-chain VM and the on-chain VM implemented on smart contract. 20 | - To ensure the efficiency of AI model inference in the VM, we have implemented a lightweight DNN library specifically designed for this purpose instead of relying on popular ML frameworks like Tensorflow or PyTorch. Additionally, a script that can convert Tensorflow and PyTorch models to this lightweight library is provided. 21 | - The cross-compilation technology has been applied to compile the AI model inference code into the VM program instructions. 22 | - The VM image is managed with a Merkle tree, only the Merkle root will be uploaded to the on-chain smart contract. (the Merkle root stands for the VM state) 23 | 24 | ![image](assets/image.png) 25 | 26 | - The bisection protocol will help to locate the dispute step, the step will be sent to the arbitration contract on the blockchain 27 | 28 | ![dispute](assets/dispute.png) 29 | 30 | **Performance**: We have tested a basic AI model (a DNN model for MNIST classification) on a PC.We are able to complete the DNN inference within 2 seconds in the VM, and the entire challenge process can be completed within 2 minutes in a local Ethereum test environment. 31 | 32 | ## Multi-Phase Verification Game 33 | 34 | ### **Limitations of One-Phase Pinpoint Protocol** 35 | 36 | The one-phase verification game has a critical drawback: all computations must be executed within the Virtual Machine (VM), preventing us from leveraging the full potential of GPU/TPU acceleration or parallel processing. Consequently, this restriction severely hampers the efficiency of large model inference, which also aligns with the current limitation of the referred delegation of computation (RDoC) protocol. 37 | 38 | ### Transitioning to a Multi-Phase Protocol 39 | 40 | To address the constraints imposed by the one-phase protocol and ensure that OPML can achieve performance levels comparable to the native environment, we propose an extension to a multi-phase protocol. With this approach, we only need to conduct the computation in the VM only in the final phase, resembling the single-phase protocol. For other phases, we have the flexibility to perform computations that lead to state transitions in the native environment, leveraging the capabilities of CPU, GPU, TPU, or even parallel processing. By reducing the reliance on the VM, we significantly minimize overhead, resulting in a remarkable enhancement in the execution performance of OPML, almost akin to that of the native environment. 41 | 42 | The following figure demonstrates a verification game consists of two phases (k = 2). In Phase-1, the process resembles that of a single-phase verification game, where each state transition corresponds to a single VM microinstruction that changes the VM state. In Phase-2, the state transition corresponds to a "Large Instruction" encompassing multiple microinstructions that change the computation context. 43 | 44 | The submitter and verifier will first start the verification game on Phase-2 using bisection protocol to locate the dispute step on a "large instruction". This step will be send to the next phase, Phase-1. Phase-1 works like the single-phase verification game. The bisection protocol in Phase-1 will help to locate the dispute step on a VM microinstruction. This step will be sent to the arbitration contract on the blockchain. 45 | 46 | To ensure the integrity and security of the transition to the next phase, we rely on the Merkle tree. This operation involves extracting a Merkle sub-tree from a higher-level phase, thus guaranteeing the seamless continuation of the verification process. 47 | 48 | 49 | 50 | ![multi-phase](assets/multi-phase.png) 51 | 52 | ### Multi-Phase OPML 53 | 54 | In this demonstration, we present a two-phase OPML approach, as utilized in the LLaMA model: 55 | 56 | - The computation process of Machine Learning (ML), specifically Deep Neural Networks (DNN), can be represented as a computation graph denoted as $G$. This graph consists of various computation nodes, capable of storing intermediate computation results. 57 | - DNN model inference is essentially a computation process on the aforementioned computation graph. The entire graph can be considered as the inference state (computation context in Phase-2). As each node is computed, the results are stored within that node, thereby advancing the computation graph to its next state. 58 | 59 | ![multi-phase](assets/computation_graph.png) 60 | 61 | - Therefore, we can first conduct the verification game on the computation graph (At phase-2). On the phase-2 verification game, the computation on nodes of the graph can be conducted in native environment using multi-thread CPU or GPU. The bisection protocol will help to locate the dispute node, and the computation of this node will be sent to the next phase (phase-1) bisection protocol. 62 | - In Phase-1 bisection, we transform the computation of a single node into Virtual Machine (VM) instructions, similar to what is done in the single-phase protocol. 63 | 64 | It is worth noting that we anticipate introducing a multi-phase OPML approach (comprising more than two phases) when the computation on a single node within the computation graph remains computationally complex. This extension will further enhance the overall efficiency and effectiveness of the verification process. 65 | 66 | ### Performance Improvement 67 | 68 | Here, we present a concise discussion and analysis of our proposed multi-phase verification framework. 69 | 70 | Suppose there are $n$ nodes in the DNN computation graph, and each node needs to take $m$ VM microinstructions to complete the calculation in VM. Assuming that the speedup ratio on the calculation on each node using GPU or parallel computing is $\alpha$. This ratio represents the acceleration achieved through GPU or parallel computing and can reach significant values, often ranging from tens to even hundreds of times faster than VM execution. 71 | 72 | Based on these considerations, we draw the following conclusions: 73 | 74 | 1. Two-phase OPML outperforms single-phase OPML, achieving a computation speedup of $\alpha$ times. The utilization of multi-phase verification enables us to take advantage of the accelerated computation capabilities offered by GPU or parallel processing, leading to substantial gains in overall performance. 75 | 2. When comparing the sizes of the Merkle trees, we find that in two-phase OPML, the size is $O(m + n)$, whereas in single-phase OPML, the size is significantly larger at $O(mn)$. The reduction in Merkle tree size further highlights the efficiency and scalability of the multi-phase design. 76 | 77 | In summary, the multi-phase verification framework presents a remarkable performance improvement, ensuring more efficient and expedited computations, particularly when leveraging the speedup capabilities of GPU or parallel processing. Additionally, the reduced Merkle tree size adds to the system's effectiveness and scalability, making multi-phase OPML a compelling choice for various applications. 78 | 79 | ## Consistency and Determinism 80 | 81 | In OPML, ensuring the consistency of ML results is of paramount importance. 82 | 83 | During the native execution of DNN computations, especially across various hardware platforms, differences in execution results may arise due to the characteristics of floating-point numbers. For instance, parallel computations involving floating-point numbers, such as $(a + b) + c$ versus $a + (b + c)$, often yield non-identical outcomes due to rounding errors. Additionally, factors such as programming language, compiler version, and operating system can influence the computed results of floating-point numbers, leading to further inconsistency in ML results. 84 | 85 | To tackle these challenges and guarantee the consistency of OPML, we employ two key approaches: 86 | 87 | 1. Fixed-point arithmetic, also known as quantization technology, is adopted. This technique enables us to represent and perform computations using fixed precision rather than floating-point numbers. By doing so, we mitigate the effects of floating-point rounding errors, leading to more reliable and consistent results. 88 | 2. We leverage software-based floating-point libraries that are designed to function consistently across different platforms. These libraries ensure cross-platform consistency and determinism of the ML results, regardless of the underlying hardware or software configurations. 89 | 90 | By combining fixed-point arithmetic and software-based floating-point libraries, we establish a robust foundation for achieving consistent and reliable ML results within the OPML framework. This harmonization of techniques enables us to overcome the inherent challenges posed by floating-point variations and platform disparities, ultimately enhancing the integrity and dependability of OPML computations. 91 | 92 | ## OPML vs ZKML 93 | 94 | | | OPML | ZKML | 95 | | -------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | 96 | | model size | any size (available for extremely large model) | small/limited (due to the cost of ZKP generation) | 97 | | validity proof | fraud proof | zero-knowledge proof (ZKP) | 98 | | training support* | √ | × | 99 | | requirement | Any PC with CPU/GPU | Large memory for ZK circuit | 100 | | Finality | Delay for challenge period | No delays | 101 | | service cost | low (inference and training can be conducted in native environment) | extremely high (generating a ZKP for ML inference is extremely high) | 102 | | security | crypto-economic incentives for security | cryptographic security | 103 | 104 | *: Within the current OPML framework, our primary focus lies on the inference of ML models, allowing for efficient and secure model computations. However, it is essential to highlight that our framework also supports the training process, making it a versatile solution for various machine learning tasks. 105 | 106 | Note that OPML is still under development. If you are interested in becoming part of this exciting initiative and contributing to the OPML project, please do not hesitate to reach out to us. 107 | -------------------------------------------------------------------------------- /docs/assets/computation_graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ora-io/opml/2fa43311a4a95bcd96f5092d0a3b34e84da49eb0/docs/assets/computation_graph.png -------------------------------------------------------------------------------- /docs/assets/dispute.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ora-io/opml/2fa43311a4a95bcd96f5092d0a3b34e84da49eb0/docs/assets/dispute.png -------------------------------------------------------------------------------- /docs/assets/image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ora-io/opml/2fa43311a4a95bcd96f5092d0a3b34e84da49eb0/docs/assets/image.png -------------------------------------------------------------------------------- /docs/assets/multi-phase.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ora-io/opml/2fa43311a4a95bcd96f5092d0a3b34e84da49eb0/docs/assets/multi-phase.png -------------------------------------------------------------------------------- /docs/tutorial.md: -------------------------------------------------------------------------------- 1 | # Tutorial 2 | 3 | In this tutorial, you will learn how to run a simple handwritten digit recognition DNN model (MNIST) in OPML. 4 | 5 | ## Train 6 | 7 | First train a DNN model using Pytorch, the training part is shown in `mlgo/examples/mnist/training/mnist.ipynb` 8 | and then save the model at `mlgo/examples/mnist/models/mnist/mnist-small.state_dict` 9 | 10 | ## Model Format Conversion 11 | 12 | Convert the Pytorch model to GGML format, only save the hyperparameters used plus the model weights and biases. Run `mlgo/examples/mnist/convert-h5-to-ggml.py` to convert your pytorch model. The output format is: 13 | 14 | - magic constant (int32) 15 | - repeated list of tensors 16 | - number of dimensions of tensor (int32) 17 | - tensor dimension (int32 repeated) 18 | - values of tensor (int32) 19 | 20 | Note that the model is saved in big-endian, making it easy to process in the big-endian MIPS-32 VM. 21 | 22 | ## Construct ML Program in MIPS VM 23 | 24 | Write a program for DNN model inference, the source code is at `mlgo/examples/mnist_mips` 25 | 26 | Note that the model, input, and output are all stored in the VM memory. 27 | 28 | Go supports compilation to MIPS. However, the generated executable is in ELF format. We'd like to get a pure sequence of MIPS instructions instead. 29 | To build a ML program in MIPS VM, just run `mlgo/examples/mnist_mips/build.sh` 30 | 31 | ## Construct VM Image 32 | 33 | The user who proposes a ML inference request should first construct an initial VM image 34 | 35 | ```shell 36 | mlvm/mlvm --outputGolden --basedir=/tmp/cannon --program="$PROGRAM_PATH" --model="$MODEL_PATH" --data="$DATA_PATH" --mipsVMCompatible 37 | ``` 38 | 39 | The initial VM image is saved at `/tmp/cannon/golden.json`, organized in a Merkle trie tree format. 40 | 41 | ## On-chain Dispute 42 | 43 | (For more details, please refer to `demo/challenge_simple.sh`) 44 | 45 | First start a local node 46 | ```shell 47 | npx hardhat node 48 | ``` 49 | Then deploy the smart contract with the initial VM image 50 | ```shell 51 | npx hardhat run scripts/deploy.js --network localhost 52 | ``` 53 | Then the challenger can start the dispute game 54 | ```shell 55 | BASEDIR=/tmp/cannon_fault npx hardhat run scripts/challenge.js --network localhost 56 | ``` 57 | Then the submitter and the challenger will interactively find the dispute point using bisection protocol 58 | ```shell 59 | for i in {1..25}; do 60 | echo "" 61 | echo "--- STEP $i / 25 ---" 62 | echo "" 63 | BASEDIR=/tmp/cannon_fault CHALLENGER=1 npx hardhat run scripts/respond.js --network localhost 64 | BASEDIR=/tmp/cannon CHALLENGER=0 npx hardhat run scripts/respond.js --network localhost 65 | done 66 | ``` 67 | Finally, the bisection protocol will help to locate the dispute step, the step will be sent to the arbitration contract on the blockchain. 68 | ```shell 69 | npx hardhat run scripts/assert.js --network localhost 70 | ``` 71 | -------------------------------------------------------------------------------- /hardhat.config.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @type import('hardhat/config').HardhatUserConfig 3 | */ 4 | 5 | require("@nomicfoundation/hardhat-toolbox"); 6 | require("@nomiclabs/hardhat-ethers"); 7 | require("hardhat-gas-reporter"); 8 | const fs = require("fs") 9 | 10 | // attempt to read private key 11 | let private = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" 12 | try { 13 | private = fs.readFileSync(process.env.HOME+"/.privatekey").toString().strip() 14 | } catch { 15 | } 16 | 17 | 18 | module.exports = { 19 | //defaultNetwork: "hosthat", 20 | networks: { 21 | l1: { 22 | url: "http://127.0.0.1:8545/", 23 | accounts: ["0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"], 24 | timeout: 600_000, 25 | }, 26 | l2: { 27 | url: "http://127.0.0.1:9545/", 28 | accounts: ["0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"], 29 | timeout: 600_000, 30 | }, 31 | }, 32 | solidity: { 33 | version: "0.7.3", 34 | settings: { 35 | optimizer: { 36 | enabled: true, 37 | runs: 200 38 | } 39 | } 40 | }, 41 | }; 42 | -------------------------------------------------------------------------------- /mlvm/go.mod: -------------------------------------------------------------------------------- 1 | module mlvm 2 | 3 | go 1.20 4 | 5 | replace github.com/ethereum/go-ethereum => github.com/ethereum-optimism/minigeth v0.0.0-20220614121031-c2b6152b4afb 6 | 7 | replace github.com/unicorn-engine/unicorn => ../unicorn 8 | 9 | replace mlgo => ../mlgo 10 | 11 | require ( 12 | mlgo v0.0.0 13 | github.com/ethereum/go-ethereum v1.10.8 14 | github.com/fatih/color v1.13.0 15 | github.com/unicorn-engine/unicorn v0.0.0-20211005173419-3fadb5aa5aad 16 | ) 17 | 18 | require ( 19 | github.com/mattn/go-colorable v0.1.13 // indirect 20 | github.com/mattn/go-isatty v0.0.17 // indirect 21 | golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect 22 | golang.org/x/sys v0.6.0 // indirect 23 | ) 24 | 25 | require ( 26 | github.com/jessevdk/go-flags v1.5.0 27 | github.com/mattn/go-colorable v0.1.13 28 | github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db 29 | github.com/schollz/progressbar/v3 v3.13.1 30 | github.com/x448/float16 v0.8.4 31 | golang.org/x/exp v0.0.0-20230321023759-10a507213a29 32 | ) 33 | 34 | require ( 35 | github.com/mattn/go-isatty v0.0.17 // indirect 36 | github.com/mattn/go-runewidth v0.0.14 // indirect 37 | github.com/rivo/uniseg v0.2.0 // indirect 38 | golang.org/x/sys v0.6.0 // indirect 39 | golang.org/x/term v0.6.0 // indirect 40 | ) -------------------------------------------------------------------------------- /mlvm/go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/ethereum-optimism/minigeth v0.0.0-20220614121031-c2b6152b4afb h1:BVFDz5VrynnyKc1OyqytK3CoELHOW+mM1rGzIJew1gY= 4 | github.com/ethereum-optimism/minigeth v0.0.0-20220614121031-c2b6152b4afb/go.mod h1:BoagkS+rYQOTLFLb/WIBDfgEu49jFt1mPZPoUuKoabE= 5 | github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= 6 | github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= 7 | github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= 8 | github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= 9 | github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= 10 | github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= 11 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 12 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 13 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 14 | github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= 15 | github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= 16 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 17 | github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= 18 | github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 19 | github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= 20 | github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 21 | github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= 22 | github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= 23 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 24 | github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= 25 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 26 | github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE= 27 | github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ= 28 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 29 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 30 | github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= 31 | github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= 32 | golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= 33 | golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 34 | golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= 35 | golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= 36 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 37 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 38 | golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 39 | golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= 40 | golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 41 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 42 | golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= 43 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 44 | golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= 45 | golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= 46 | -------------------------------------------------------------------------------- /mlvm/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "mlvm/vm" 4 | 5 | func main() { 6 | vm.Run() 7 | } -------------------------------------------------------------------------------- /mlvm/vm/ml.go: -------------------------------------------------------------------------------- 1 | package vm 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | 7 | llama "mlgo/examples/llama/llama_go" 8 | "mlgo/examples/mnist" 9 | "mlgo/ml" 10 | ) 11 | 12 | 13 | func LLAMA(nodeID int) ([]byte, int, error){ 14 | modelFile := "/path/models/llama-7b-fp32.bin.2" 15 | prompt := "Why Golang is so popular?" 16 | threadCount := 32 17 | ctx, err := llama.LoadModel(modelFile, true) 18 | fmt.Println("Load Model Finish") 19 | if err != nil { 20 | fmt.Println("load model error: ", err) 21 | return nil, 0, err 22 | } 23 | embd := ml.Tokenize(ctx.Vocab, prompt, true) 24 | graph, mlctx, err := llama.ExpandGraph(ctx, embd, uint32(len(embd)), 0, threadCount) 25 | ml.GraphComputeByNodes(mlctx, graph, nodeID) 26 | envBytes := ml.SaveComputeNodeEnvToBytes(uint32(nodeID), graph.Nodes[nodeID], graph, true) 27 | return envBytes, int(graph.NodesCount), nil 28 | } 29 | 30 | func MNIST(nodeID int) ([]byte, int, error) { 31 | threadCount := 1 32 | modelFile := "../../mlgo/examples/mnist/models/mnist/ggml-model-small-f32.bin" 33 | model, err := mnist.LoadModel(modelFile) 34 | if err != nil { 35 | fmt.Println("Load model error: ", err) 36 | return nil, 0, err 37 | } 38 | // load input 39 | input, err := MNIST_Input(false) 40 | if err != nil { 41 | fmt.Println("Load input data error: ", err) 42 | return nil, 0, err 43 | } 44 | graph, ctx := mnist.ExpandGraph(model, threadCount, input) 45 | ml.GraphComputeByNodes(ctx, graph, nodeID) 46 | envBytes := ml.SaveComputeNodeEnvToBytes(uint32(nodeID), graph.Nodes[nodeID], graph, true) 47 | return envBytes, int(graph.NodesCount), nil 48 | } 49 | 50 | func MNIST_Input(show bool) ([]float32, error) { 51 | dataFile := "../../mlgo/examples/mnist/models/mnist/input_7" 52 | buf, err := ioutil.ReadFile(dataFile) 53 | if err != nil { 54 | fmt.Println(err) 55 | return nil, err 56 | } 57 | digits := make([]float32, 784) 58 | 59 | // render the digit in ASCII 60 | var c string 61 | for row := 0; row < 28; row++{ 62 | for col := 0; col < 28; col++ { 63 | digits[row*28 + col] = float32(buf[row*28 + col]) 64 | if buf[row*28 + col] > 230 { 65 | c += "*" 66 | } else { 67 | c += "_" 68 | } 69 | } 70 | c += "\n" 71 | } 72 | if show { 73 | fmt.Println(c) 74 | } 75 | 76 | 77 | return digits, nil 78 | } -------------------------------------------------------------------------------- /mlvm/vm/ml_test.go: -------------------------------------------------------------------------------- 1 | package vm 2 | 3 | import ( 4 | "fmt" 5 | llama "mlgo/examples/llama/llama_go" 6 | "mlgo/examples/mnist" 7 | "mlgo/ml" 8 | "testing" 9 | ) 10 | 11 | func TestMNIST(t *testing.T) { 12 | threadCount := 1 13 | modelFile := "../../mlgo/examples/mnist/models/mnist/ggml-model-small-f32.bin" 14 | model, err := mnist.LoadModel(modelFile) 15 | if err != nil { 16 | fmt.Println("Load model error: ", err) 17 | return 18 | } 19 | // load input 20 | input, err := MNIST_Input(true) 21 | if err != nil { 22 | fmt.Println("Load input data error: ", err) 23 | return 24 | } 25 | graph, _ := mnist.ExpandGraph(model, threadCount, input) 26 | fmt.Println("graph.nodeNum: ", graph.NodesCount) 27 | } 28 | 29 | func TestLLAMA(t *testing.T) { 30 | modelFile := "/path/models/llama-7b-fp32.bin.2" 31 | prompt := "Why Golang is so popular?" 32 | threadCount := 32 33 | ctx, err := llama.LoadModel(modelFile, true) 34 | fmt.Println("Load Model Finish") 35 | if err != nil { 36 | fmt.Println("load model error: ", err) 37 | return 38 | } 39 | embd := ml.Tokenize(ctx.Vocab, prompt, true) 40 | graph, _, _ := llama.ExpandGraph(ctx, embd, uint32(len(embd)), 0, threadCount) 41 | fmt.Println("graph.nodeCount: ", graph.NodesCount) 42 | } -------------------------------------------------------------------------------- /mlvm/vm/run_mlgo_test.go: -------------------------------------------------------------------------------- 1 | package vm 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "fmt" 7 | "io/ioutil" 8 | "reflect" 9 | "testing" 10 | "time" 11 | 12 | uc "github.com/unicorn-engine/unicorn/bindings/go/unicorn" 13 | ) 14 | 15 | 16 | 17 | func LoadMNISTData(mu uc.Unicorn, file string, ram map[uint32](uint32)) error { 18 | // load a random test digit 19 | buf, err := ioutil.ReadFile(file) 20 | if err != nil { 21 | fmt.Println(err) 22 | return err 23 | } 24 | 25 | // fin, err := os.Open(file) 26 | // if err != nil { 27 | // fmt.Println(err) 28 | // return err 29 | // } 30 | // buf := make([]byte, 784) 31 | // if count, err := fin.Read(buf); err != nil || count != int(len(buf)) { 32 | // fmt.Println(err, count) 33 | // return err 34 | // } 35 | 36 | // // render the digit in ASCII 37 | // for row := 0; row < 28; row++{ 38 | // for col := 0; col < 28; col++ { 39 | // var c string 40 | // if buf[row*28 + col] > 230 { 41 | // c = "*" 42 | // } else { 43 | // c = "_" 44 | // } 45 | // fmt.Printf(c) 46 | // } 47 | // fmt.Println("") 48 | // } 49 | // fmt.Println("") 50 | 51 | //buf is the data 52 | inputSize := len(buf) 53 | LoadBytesToUnicorn(mu, IntToBytes(inputSize), ram, INPUT_ADDR) 54 | LoadBytesToUnicorn(mu, buf, ram, INPUT_ADDR + 4) 55 | 56 | return nil 57 | } 58 | 59 | // testing 60 | func TestMLGo_MNIST(t *testing.T){ 61 | fn := "../../mlgo/mlgo.bin" 62 | fn = "../../mlgo/examples/mnist_mips/mlgo.bin" 63 | // fn = "../../Rollup_DL/mipigo/test/test2.bin" //for testing 64 | modelLittleEndianFile := "../../mlgo/examples/mnist/models/mnist/ggml-model-small-f32.bin" 65 | modelBigEndianFile := "../../mlgo/examples/mnist/models/mnist/ggml-model-small-f32-big-endian.bin" 66 | dataFile := "../../mlgo/examples/mnist/models/mnist/t10k-images.idx3-ubyte" 67 | dataFile = "../../mlgo/examples/mnist/models/mnist/input_7" 68 | steps := 1000000000 69 | // steps = 12066041 70 | 71 | // reachFinalState := true 72 | 73 | modelFile := modelLittleEndianFile 74 | if READ_FROM_BIDENDIAN { 75 | modelFile = modelBigEndianFile 76 | } 77 | 78 | totalSteps := 0; 79 | 80 | ram := make(map[uint32](uint32)) 81 | 82 | callback := func(step int, mu uc.Unicorn, ram map[uint32](uint32)) { 83 | totalSteps += 1; 84 | // sync at each step is very slow 85 | // SyncRegs(mu, ram) 86 | if step%10000000 == 0 { 87 | steps_per_sec := float64(step) * 1e9 / float64(time.Now().Sub(ministart).Nanoseconds()) 88 | fmt.Printf("%10d pc: %x steps per s %f ram entries %d\n", step, ram[0xc0000080], steps_per_sec, len(ram)) 89 | } 90 | // halt at steps 91 | if step == steps { 92 | fmt.Println("what what what ? final!") 93 | // reachFinalState = false 94 | mu.RegWrite(uc.MIPS_REG_PC, 0x5ead0004) 95 | } 96 | } 97 | 98 | mu := GetHookedUnicorn("", ram, callback) 99 | // program 100 | ZeroRegisters(ram) 101 | LoadMappedFileUnicorn(mu, fn, ram, 0) 102 | // load model and input 103 | LoadModel(mu, modelFile, ram) 104 | LoadMNISTData(mu, dataFile, ram) 105 | 106 | // initial checkpoint 107 | // WriteCheckpoint(ram, "/tmp/cannon/golden.json", 0) 108 | 109 | SyncRegs(mu, ram) 110 | mu.Start(0, 0x5ead0004) 111 | SyncRegs(mu, ram) 112 | 113 | // final checkpoint 114 | // if reachFinalState { 115 | // WriteCheckpoint(ram, fmt.Sprintf("/tmp/cannon/checkpoint_%d.json", totalSteps), totalSteps) 116 | // } 117 | WriteCheckpoint(ram, "/tmp/cannon/checkpoint_final.json", totalSteps) 118 | 119 | SyncRegs(mu, ram) 120 | 121 | fmt.Println("ram[0x32000000]: ", ram[0x32000000]) 122 | fmt.Println("ram[0x32000004]: ", ram[0x32000004]) 123 | fmt.Printf("PC: %x\n", ram[0xC0000080]) 124 | 125 | fmt.Println("total steps: ", totalSteps) 126 | } 127 | 128 | 129 | // test on node 130 | func TestMLGo_MNIST_Node(t *testing.T){ 131 | fn := "../../mlgo/ml_mips/ml_mips.bin" 132 | dataFile := "../../mlgo/examples/mnist/models/mnist/node_5" 133 | dataFile = "../../mlgo/examples/llama/data/node_1253" 134 | steps := 1000000000 135 | // steps = 12066041 136 | 137 | 138 | totalSteps := 0; 139 | 140 | ram := make(map[uint32](uint32)) 141 | 142 | callback := func(step int, mu uc.Unicorn, ram map[uint32](uint32)) { 143 | totalSteps += 1; 144 | // sync at each step is very slow 145 | // SyncRegs(mu, ram) 146 | if step%10000000 == 0 { 147 | steps_per_sec := float64(step) * 1e9 / float64(time.Now().Sub(ministart).Nanoseconds()) 148 | fmt.Printf("%10d pc: %x steps per s %f ram entries %d\n", step, ram[0xc0000080], steps_per_sec, len(ram)) 149 | } 150 | // halt at steps 151 | if step == steps { 152 | fmt.Println("what what what ? final!") 153 | // reachFinalState = false 154 | mu.RegWrite(uc.MIPS_REG_PC, 0x5ead0004) 155 | } 156 | } 157 | 158 | mu := GetHookedUnicorn("", ram, callback) 159 | // program 160 | ZeroRegisters(ram) 161 | LoadMappedFileUnicorn(mu, fn, ram, 0) 162 | // load model and input 163 | LoadInputData(mu, dataFile, ram) 164 | 165 | // initial checkpoint 166 | // WriteCheckpoint(ram, "/tmp/cannon/golden.json", 0) 167 | 168 | SyncRegs(mu, ram) 169 | mu.Start(0, 0x5ead0004) 170 | SyncRegs(mu, ram) 171 | 172 | // final checkpoint 173 | // if reachFinalState { 174 | // WriteCheckpoint(ram, fmt.Sprintf("/tmp/cannon/checkpoint_%d.json", totalSteps), totalSteps) 175 | // } 176 | WriteCheckpoint(ram, "/tmp/cannon/checkpoint_final.json", totalSteps) 177 | 178 | SyncRegs(mu, ram) 179 | 180 | fmt.Println("total steps: ", totalSteps) 181 | } 182 | 183 | func BytesToInt32(b []byte, isBigEndian bool) uint32 { 184 | bytesBuffer := bytes.NewBuffer(b) 185 | 186 | var x uint32 187 | if isBigEndian { 188 | binary.Read(bytesBuffer, binary.BigEndian, &x) 189 | } else { 190 | binary.Read(bytesBuffer, binary.LittleEndian, &x) 191 | } 192 | 193 | 194 | return x 195 | } 196 | 197 | // Faster design! we do not store 0 in ram and trie 198 | func TestMLGo_MNIST2_Fast(t *testing.T){ 199 | fn := "../../mlgo/mlgo.bin" 200 | // fn = "../../Rollup_DL/mipigo/test/test2.bin" //for testing 201 | modelLittleEndianFile := "../../mlgo/examples/mnist/models/mnist/ggml-model-small-f32.bin" 202 | modelBigEndianFile := "../../mlgo/examples/mnist/models/mnist/ggml-model-f32-big-endian.bin" 203 | dataFile := "../../mlgo/examples/mnist/models/mnist/t10k-images.idx3-ubyte" 204 | dataFile = "../../mlgo/examples/mnist/models/mnist/input_7" 205 | steps := 1000000000 206 | calSteps := false 207 | // steps = 12066041 208 | 209 | // reachFinalState := true 210 | 211 | modelFile := modelLittleEndianFile 212 | if READ_FROM_BIDENDIAN { 213 | modelFile = modelBigEndianFile 214 | } 215 | 216 | totalSteps := 0; 217 | 218 | ram := make(map[uint32](uint32)) 219 | 220 | 221 | mu := GetHookedUnicorn("", ram, nil) 222 | 223 | if calSteps { 224 | mu.HookAdd(uc.HOOK_CODE, func(mu uc.Unicorn, addr uint64, size uint32) { 225 | totalSteps += 1 226 | }, 0, 0x80000000) 227 | } 228 | 229 | 230 | // program 231 | ZeroRegisters(ram) 232 | LoadMappedFileUnicorn(mu, fn, ram, 0) 233 | // load model and input 234 | LoadModel(mu, modelFile, ram) 235 | LoadMNISTData(mu, dataFile, ram) 236 | 237 | SyncRegs(mu, ram) 238 | 239 | option := &uc.UcOptions{Timeout: 0, Count: uint64(steps)} 240 | mu.StartWithOptions(0, 0x5ead0004, option) 241 | 242 | // mu.RegWrite(uc.MIPS_REG_PC, 0x5ead0004) 243 | SyncRegs(mu, ram) 244 | 245 | memory, err := mu.MemRead(0,0x80000000-4) 246 | if err != nil { 247 | fmt.Println(err) 248 | } 249 | cnt := 0 250 | for i := 0; i < len(memory)-4; i += 4 { 251 | if memory[i] == 0 && memory[i+1] ==0 && memory[i+2] == 0 && memory[i+3] == 0 { 252 | continue 253 | } 254 | v := BytesToInt32(memory[i:i+4], true) 255 | if v != 0 { 256 | cnt += 1 257 | ram[uint32(i)] = v 258 | } 259 | } 260 | 261 | for k,v := range ram { 262 | if v == 0{ 263 | delete(ram, k) 264 | } 265 | } 266 | 267 | fmt.Printf("cnt: %d, size of ram: %d\n", cnt, len(ram)) 268 | // final checkpoint 269 | // if reachFinalState { 270 | // WriteCheckpoint(ram, fmt.Sprintf("/tmp/cannon/checkpoint_%d.json", totalSteps), totalSteps) 271 | // } 272 | WriteCheckpoint(ram, "/tmp/cannon/checkpoint_final_test.json", totalSteps) 273 | 274 | 275 | fmt.Println("ram[0x5ead0004]: ", ram[0x5ead0004]) 276 | fmt.Println("ram[0x32000000]: ", ram[0x32000000]) 277 | fmt.Println("ram[0x32000004]: ", ram[0x32000004]) 278 | 279 | fmt.Println("total steps: ", totalSteps) 280 | 281 | } 282 | 283 | 284 | func TestMLGo_MNIST2(t *testing.T){ 285 | fn := "../../mlgo/mlgo.bin" 286 | // fn = "../../Rollup_DL/mipigo/test/test2.bin" //for testing 287 | modelLittleEndianFile := "../../mlgo/examples/mnist/models/mnist/ggml-model-small-f32.bin" 288 | modelBigEndianFile := "../../mlgo/examples/mnist/models/mnist/ggml-model-small-f32-big-endian.bin" 289 | dataFile := "../../mlgo/examples/mnist/models/mnist/t10k-images.idx3-ubyte" 290 | dataFile = "../../mlgo/examples/mnist/models/mnist/input_7" 291 | steps := 1000000000 292 | compare := true 293 | // steps = 12066041 294 | 295 | // reachFinalState := true 296 | 297 | modelFile := modelLittleEndianFile 298 | if READ_FROM_BIDENDIAN { 299 | modelFile = modelBigEndianFile 300 | } 301 | 302 | totalSteps := 0; 303 | 304 | ram := make(map[uint32](uint32)) 305 | 306 | 307 | mu := GetHookedUnicorn("", ram, nil) 308 | 309 | mu.HookAdd(uc.HOOK_CODE, func(mu uc.Unicorn, addr uint64, size uint32) { 310 | totalSteps += 1 311 | }, 0, 0x80000000) 312 | 313 | // program 314 | ZeroRegisters(ram) 315 | LoadMappedFileUnicorn(mu, fn, ram, 0) 316 | // load model and input 317 | LoadModel(mu, modelFile, ram) 318 | LoadMNISTData(mu, dataFile, ram) 319 | 320 | SyncRegs(mu, ram) 321 | 322 | option := &uc.UcOptions{Timeout: 0, Count: uint64(steps)} 323 | mu.StartWithOptions(0, 0x5ead0004, option) 324 | 325 | // mu.RegWrite(uc.MIPS_REG_PC, 0x5ead0004) 326 | SyncRegs(mu, ram) 327 | 328 | memory, err := mu.MemRead(0,0x80000000-4) 329 | fmt.Println("memory len: ", len(memory)) 330 | if err != nil { 331 | fmt.Println(err) 332 | } 333 | cnt := 0 334 | for i := 0; i < len(memory)-4; i += 4 { 335 | if memory[i] == 0 && memory[i+1] ==0 && memory[i+2] == 0 && memory[i+3] == 0 { 336 | continue 337 | } 338 | v := BytesToInt32(memory[i:i+4], true) 339 | if v != 0 { 340 | cnt += 1 341 | if ram_v, ok := ram[uint32(i)]; !ok || ram_v == v { 342 | ram[uint32(i)] = v 343 | } else { 344 | fmt.Printf("ram address: %d updates from %d to %d\n", i, ram_v, v) 345 | } 346 | ram[uint32(i)] = v 347 | } 348 | } 349 | 350 | for k,v := range ram { 351 | if v == 0{ 352 | delete(ram, k) 353 | } 354 | } 355 | 356 | fmt.Printf("cnt: %d, size of ram: %d\n", cnt, len(ram)) 357 | // final checkpoint 358 | // if reachFinalState { 359 | // WriteCheckpoint(ram, fmt.Sprintf("/tmp/cannon/checkpoint_%d.json", totalSteps), totalSteps) 360 | // } 361 | WriteCheckpoint(ram, "/tmp/cannon/checkpoint_final_test.json", totalSteps) 362 | 363 | 364 | fmt.Println("ram[0x5ead0004]: ", ram[0x5ead0004]) 365 | fmt.Println("ram[0x32000000]: ", ram[0x32000000]) 366 | fmt.Println("ram[0x32000004]: ", ram[0x32000004]) 367 | 368 | fmt.Println("total steps: ", totalSteps) 369 | 370 | // the difference happens because we do not delete the memory with 0 value! 371 | 372 | // compare 373 | if compare { 374 | steps = 0 375 | heap_start = 0 376 | old_ram := MLGo_MNIST2_helper() 377 | fmt.Printf("old ram len: %d, new ram len: %d\n", len(old_ram), len(ram)) 378 | 379 | for k,v := range ram { 380 | old_v, ok := old_ram[k] 381 | if old_v != v || !ok{ 382 | fmt.Printf("diff! In RAM! address: %x, new value: %d, old value: %d\n", k, v, old_v) 383 | } 384 | } 385 | for k,old_v := range old_ram { 386 | v, ok := ram[k] 387 | if old_v != v || !ok{ 388 | fmt.Printf("diff! In OldRam! address: %x, new value: %d, old value: %d\n", k, v, old_v) 389 | } 390 | } 391 | if reflect.DeepEqual(ram, old_ram) { 392 | fmt.Println("equal") 393 | } else { 394 | fmt.Println("non-equal") 395 | } 396 | } 397 | } 398 | 399 | func MLGo_MNIST2_helper() map[uint32](uint32){ 400 | fn := "../../mlgo/mlgo.bin" 401 | // fn = "../../Rollup_DL/mipigo/test/test2.bin" //for testing 402 | modelLittleEndianFile := "../../mlgo/examples/mnist/models/mnist/ggml-model-small-f32.bin" 403 | modelBigEndianFile := "../../mlgo/examples/mnist/models/mnist/ggml-model-small-f32-big-endian.bin" 404 | dataFile := "../../mlgo/examples/mnist/models/mnist/t10k-images.idx3-ubyte" 405 | dataFile = "../../mlgo/examples/mnist/models/mnist/input_7" 406 | steps := 1000000000 407 | // steps = 12066041 408 | 409 | // reachFinalState := true 410 | 411 | modelFile := modelLittleEndianFile 412 | if READ_FROM_BIDENDIAN { 413 | modelFile = modelBigEndianFile 414 | } 415 | 416 | totalSteps := 0; 417 | 418 | ram := make(map[uint32](uint32)) 419 | 420 | callback := func(step int, mu uc.Unicorn, ram map[uint32](uint32)) { 421 | totalSteps += 1; 422 | // sync at each step is very slow 423 | // SyncRegs(mu, ram) 424 | if step%10000000 == 0 { 425 | steps_per_sec := float64(step) * 1e9 / float64(time.Now().Sub(ministart).Nanoseconds()) 426 | fmt.Printf("%10d pc: %x steps per s %f ram entries %d\n", step, ram[0xc0000080], steps_per_sec, len(ram)) 427 | } 428 | // halt at steps 429 | if step == steps { 430 | fmt.Println("what what what ? final!") 431 | // reachFinalState = false 432 | mu.RegWrite(uc.MIPS_REG_PC, 0x5ead0004) 433 | } 434 | } 435 | 436 | mu := GetHookedUnicorn("", ram, callback) 437 | // program 438 | ZeroRegisters(ram) 439 | LoadMappedFileUnicorn(mu, fn, ram, 0) 440 | // load model and input 441 | LoadModel(mu, modelFile, ram) 442 | LoadMNISTData(mu, dataFile, ram) 443 | 444 | // initial checkpoint 445 | // WriteCheckpoint(ram, "/tmp/cannon/golden.json", 0) 446 | 447 | SyncRegs(mu, ram) 448 | mu.Start(0, 0x5ead0004) 449 | SyncRegs(mu, ram) 450 | 451 | for k,v := range ram { 452 | if v == 0{ 453 | delete(ram, k) 454 | } 455 | } 456 | 457 | fmt.Println("ram[0x5ead0004]: ", ram[0x5ead0004]) 458 | fmt.Println("ram[0x32000000]: ", ram[0x32000000]) 459 | fmt.Println("ram[0x32000004]: ", ram[0x32000004]) 460 | 461 | fmt.Println("total steps: ", totalSteps) 462 | 463 | return ram 464 | } 465 | 466 | 467 | func TestMLGo_MNIST3(t *testing.T){ 468 | fn := "../../mlgo/mlgo.bin" 469 | // fn = "../../Rollup_DL/mipigo/test/test2.bin" //for testing 470 | modelLittleEndianFile := "../../mlgo/examples/mnist/models/mnist/ggml-model-small-f32.bin" 471 | modelBigEndianFile := "../../mlgo/examples/mnist/models/mnist/ggml-model-small-f32-big-endian.bin" 472 | dataFile := "../../mlgo/examples/mnist/models/mnist/t10k-images.idx3-ubyte" 473 | dataFile = "../../mlgo/examples/mnist/models/mnist/input_7" 474 | steps := 1000000000 475 | // steps = 12066041 476 | 477 | // reachFinalState := true 478 | 479 | modelFile := modelLittleEndianFile 480 | if READ_FROM_BIDENDIAN { 481 | modelFile = modelBigEndianFile 482 | } 483 | 484 | totalSteps := 0; 485 | 486 | ram := make(map[uint32](uint32)) 487 | 488 | callback := func(step int, mu uc.Unicorn, ram map[uint32](uint32)) { 489 | totalSteps += 1; 490 | // sync at each step is very slow 491 | // SyncRegs(mu, ram) 492 | if step%10000000 == 0 { 493 | steps_per_sec := float64(step) * 1e9 / float64(time.Now().Sub(ministart).Nanoseconds()) 494 | fmt.Printf("%10d pc: %x steps per s %f ram entries %d\n", step, ram[0xc0000080], steps_per_sec, len(ram)) 495 | } 496 | // halt at steps 497 | if step == steps { 498 | fmt.Println("what what what ? final!") 499 | // reachFinalState = false 500 | mu.RegWrite(uc.MIPS_REG_PC, 0x5ead0004) 501 | } 502 | } 503 | 504 | mu := GetHookedUnicorn("", ram, callback) 505 | // program 506 | ZeroRegisters(ram) 507 | LoadMappedFileUnicorn(mu, fn, ram, 0) 508 | // load model and input 509 | LoadModel(mu, modelFile, ram) 510 | LoadMNISTData(mu, dataFile, ram) 511 | 512 | // initial checkpoint 513 | // WriteCheckpoint(ram, "/tmp/cannon/golden.json", 0) 514 | 515 | SyncRegs(mu, ram) 516 | mu.Start(0, 0x5ead0004) 517 | SyncRegs(mu, ram) 518 | 519 | // final checkpoint 520 | // if reachFinalState { 521 | // WriteCheckpoint(ram, fmt.Sprintf("/tmp/cannon/checkpoint_%d.json", totalSteps), totalSteps) 522 | // } 523 | 524 | // if we delete with 0 value, it should be the same 525 | for k,v := range ram { 526 | if v == 0{ 527 | delete(ram, k) 528 | } 529 | } 530 | 531 | WriteCheckpoint(ram, "/tmp/cannon/checkpoint_final.json", totalSteps) 532 | 533 | 534 | fmt.Println("ram[0x32000000]: ", ram[0x32000000]) 535 | fmt.Println("ram[0x32000004]: ", ram[0x32000004]) 536 | 537 | fmt.Println("total steps: ", totalSteps) 538 | } 539 | -------------------------------------------------------------------------------- /mlvm/vm/run_unicorn.go: -------------------------------------------------------------------------------- 1 | package vm 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "fmt" 7 | "io/ioutil" 8 | "log" 9 | "os" 10 | 11 | "github.com/ethereum/go-ethereum/common" 12 | "github.com/fatih/color" 13 | uc "github.com/unicorn-engine/unicorn/bindings/go/unicorn" 14 | ) 15 | 16 | // SHOULD BE GO BUILTIN 17 | func check(err error) { 18 | if err != nil { 19 | log.Fatal(err) 20 | } 21 | } 22 | 23 | var steps int = 0 24 | var heap_start uint64 = 0 25 | 26 | func WriteBytes(fd int, bytes []byte) { 27 | printer := color.New(color.FgWhite).SprintFunc() 28 | if fd == 1 { 29 | printer = color.New(color.FgGreen).SprintFunc() 30 | } else if fd == 2 { 31 | printer = color.New(color.FgRed).SprintFunc() 32 | } 33 | os.Stderr.WriteString(printer(string(bytes))) 34 | } 35 | 36 | func WriteRam(ram map[uint32](uint32), addr uint32, value uint32) { 37 | // we no longer delete from ram, since deleting from tries is hard 38 | if value == 0 && false { 39 | delete(ram, addr) 40 | } else { 41 | /*if addr < 0xc0000000 { 42 | fmt.Printf("store %x = %x\n", addr, value) 43 | }*/ 44 | ram[addr] = value 45 | } 46 | } 47 | 48 | var REG_OFFSET uint32 = 0xc0000000 49 | var REG_PC uint32 = REG_OFFSET + 0x20*4 50 | var REG_HEAP uint32 = REG_OFFSET + 0x23*4 51 | 52 | func SyncRegs(mu uc.Unicorn, ram map[uint32](uint32)) { 53 | pc, _ := mu.RegRead(uc.MIPS_REG_PC) 54 | //fmt.Printf("%d uni %x\n", step, pc) 55 | WriteRam(ram, 0xc0000080, uint32(pc)) 56 | 57 | addr := uint32(0xc0000000) 58 | for i := uc.MIPS_REG_ZERO; i < uc.MIPS_REG_ZERO+32; i++ { 59 | reg, _ := mu.RegRead(i) 60 | WriteRam(ram, addr, uint32(reg)) 61 | addr += 4 62 | } 63 | 64 | reg_hi, _ := mu.RegRead(uc.MIPS_REG_HI) 65 | reg_lo, _ := mu.RegRead(uc.MIPS_REG_LO) 66 | WriteRam(ram, REG_OFFSET+0x21*4, uint32(reg_hi)) 67 | WriteRam(ram, REG_OFFSET+0x22*4, uint32(reg_lo)) 68 | 69 | WriteRam(ram, REG_HEAP, uint32(heap_start)) 70 | } 71 | 72 | func GetHookedUnicorn(root string, ram map[uint32](uint32), callback func(int, uc.Unicorn, map[uint32](uint32))) uc.Unicorn { 73 | mu, err := uc.NewUnicorn(uc.ARCH_MIPS, uc.MODE_32|uc.MODE_BIG_ENDIAN) 74 | check(err) 75 | 76 | _, outputfault := os.LookupEnv("OUTPUTFAULT") 77 | 78 | mu.HookAdd(uc.HOOK_INTR, func(mu uc.Unicorn, intno uint32) { 79 | if intno != 17 { 80 | log.Fatal("invalid interrupt ", intno, " at step ", steps) 81 | } 82 | syscall_no, _ := mu.RegRead(uc.MIPS_REG_V0) 83 | v0 := uint64(0) 84 | if syscall_no == 4020 { 85 | // load the preimage from blockchain data 86 | // but for pure execution, we may not need it 87 | // unless we need to load some data, for example, parameters in DNN? 88 | oracle_hash, _ := mu.MemRead(0x30001000, 0x20) 89 | hash := common.BytesToHash(oracle_hash) 90 | key := fmt.Sprintf("%s/%s", root, hash) 91 | value, err := ioutil.ReadFile(key) 92 | // check(err) 93 | if err == nil { 94 | tmp := []byte{0, 0, 0, 0} 95 | binary.BigEndian.PutUint32(tmp, uint32(len(value))) 96 | mu.MemWrite(0x31000000, tmp) 97 | mu.MemWrite(0x31000004, value) 98 | 99 | WriteRam(ram, 0x31000000, uint32(len(value))) 100 | value = append(value, 0, 0, 0) 101 | for i := uint32(0); i < ram[0x31000000]; i += 4 { 102 | WriteRam(ram, 0x31000004+i, binary.BigEndian.Uint32(value[i:i+4])) 103 | } 104 | } 105 | 106 | } else if syscall_no == 4004 { 107 | fd, _ := mu.RegRead(uc.MIPS_REG_A0) 108 | buf, _ := mu.RegRead(uc.MIPS_REG_A1) 109 | count, _ := mu.RegRead(uc.MIPS_REG_A2) 110 | bytes, _ := mu.MemRead(buf, count) 111 | WriteBytes(int(fd), bytes) 112 | } else if syscall_no == 4090 { 113 | a0, _ := mu.RegRead(uc.MIPS_REG_A0) 114 | sz, _ := mu.RegRead(uc.MIPS_REG_A1) 115 | if a0 == 0 { 116 | v0 = 0x20000000 + heap_start 117 | heap_start += sz 118 | } else { 119 | v0 = a0 120 | } 121 | } else if syscall_no == 4045 { 122 | v0 = 0x40000000 123 | } else if syscall_no == 4120 { 124 | v0 = 1 125 | } else if syscall_no == 4246 { 126 | // exit group 127 | mu.RegWrite(uc.MIPS_REG_PC, 0x5ead0000) 128 | } else { 129 | //fmt.Println("syscall", syscall_no) 130 | } 131 | mu.RegWrite(uc.MIPS_REG_V0, v0) 132 | mu.RegWrite(uc.MIPS_REG_A3, 0) 133 | }, 0, 0) 134 | 135 | if callback != nil { 136 | mu.HookAdd(uc.HOOK_MEM_WRITE, func(mu uc.Unicorn, access int, addr64 uint64, size int, value int64) { 137 | rt := value 138 | rs := addr64 & 3 139 | addr := uint32(addr64 & 0xFFFFFFFC) 140 | if outputfault && addr == 0x30000804 { 141 | fmt.Printf("injecting output fault over %x\n", rt) 142 | rt = 0xbabababa 143 | } 144 | //fmt.Printf("%X(%d) = %x (at step %d)\n", addr, size, value, steps) 145 | if size == 1 { 146 | mem := ram[addr] 147 | val := uint32((rt & 0xFF) << (24 - (rs&3)*8)) 148 | mask := 0xFFFFFFFF ^ uint32(0xFF<<(24-(rs&3)*8)) 149 | WriteRam(ram, uint32(addr), (mem&mask)|val) 150 | } else if size == 2 { 151 | mem := ram[addr] 152 | val := uint32((rt & 0xFFFF) << (16 - (rs&2)*8)) 153 | mask := 0xFFFFFFFF ^ uint32(0xFFFF<<(16-(rs&2)*8)) 154 | WriteRam(ram, uint32(addr), (mem&mask)|val) 155 | } else if size == 4 { 156 | WriteRam(ram, uint32(addr), uint32(rt)) 157 | } else { 158 | log.Fatal("bad size write to ram") 159 | } 160 | 161 | }, 0, 0x80000000) 162 | 163 | mu.HookAdd(uc.HOOK_CODE, func(mu uc.Unicorn, addr uint64, size uint32) { 164 | callback(steps, mu, ram) 165 | steps += 1 166 | }, 0, 0x80000000) 167 | } 168 | 169 | check(mu.MemMap(0, 0x80000000)) 170 | return mu 171 | } 172 | 173 | func LoadMappedFileUnicorn(mu uc.Unicorn, fn string, ram map[uint32](uint32), base uint32) { 174 | dat, err := ioutil.ReadFile(fn) 175 | check(err) 176 | LoadData(dat, ram, base) 177 | mu.MemWrite(uint64(base), dat) 178 | } 179 | 180 | func LoadBytesToUnicorn(mu uc.Unicorn, dat []byte, ram map[uint32](uint32), base uint32) { 181 | LoadData(dat, ram, base) 182 | mu.MemWrite(uint64(base), dat) 183 | } 184 | 185 | // reimplement simple.py in go 186 | func RunUnicorn(fn string, ram map[uint32](uint32), checkIO bool, callback func(int, uc.Unicorn, map[uint32](uint32))) { 187 | root := "/tmp/cannon/0_13284469" 188 | mu := GetHookedUnicorn(root, ram, callback) 189 | 190 | // loop forever to match EVM 191 | //mu.MemMap(0x5ead0000, 0x1000) 192 | //mu.MemWrite(0xdead0000, []byte{0x08, 0x10, 0x00, 0x00}) 193 | 194 | // program 195 | dat, _ := ioutil.ReadFile(fn) 196 | mu.MemWrite(0, dat) 197 | 198 | // inputs 199 | inputs, err := ioutil.ReadFile(fmt.Sprintf("%s/input", root)) 200 | // check(err) 201 | if err == nil { 202 | mu.MemWrite(0x30000000, inputs[0:0xc0]) 203 | 204 | // load into ram 205 | LoadData(dat, ram, 0) 206 | if checkIO { 207 | LoadData(inputs[0:0xc0], ram, 0x30000000) 208 | } 209 | 210 | mu.Start(0, 0x5ead0004) 211 | } else { 212 | // load into ram 213 | LoadData(dat, ram, 0) 214 | mu.Start(0, 0x5ead0004) 215 | } 216 | 217 | 218 | 219 | if checkIO { 220 | outputs, err := ioutil.ReadFile(fmt.Sprintf("%s/output", root)) 221 | // check(err) 222 | if err == nil { 223 | real := append([]byte{0x13, 0x37, 0xf0, 0x0d}, outputs...) 224 | output, _ := mu.MemRead(0x30000800, 0x44) 225 | if bytes.Compare(real, output) != 0 { 226 | log.Fatal("mismatch output") 227 | } else { 228 | fmt.Println("output match") 229 | } 230 | } 231 | } 232 | } 233 | -------------------------------------------------------------------------------- /mlvm/vm/trie.go: -------------------------------------------------------------------------------- 1 | package vm 2 | 3 | import ( 4 | "encoding/binary" 5 | "encoding/json" 6 | "fmt" 7 | "sort" 8 | "strings" 9 | 10 | "github.com/ethereum/go-ethereum/common" 11 | "github.com/ethereum/go-ethereum/crypto" 12 | "github.com/ethereum/go-ethereum/oracle" 13 | "github.com/ethereum/go-ethereum/rlp" 14 | "github.com/ethereum/go-ethereum/trie" 15 | ) 16 | 17 | type PreimageKeyValueWriter struct{} 18 | 19 | var Preimages = make(map[common.Hash][]byte) 20 | 21 | type Jtree struct { 22 | Root common.Hash `json:"root"` 23 | Step int `json:"step"` 24 | NodeID int `json:"nodeid"` 25 | NodeCount int `json:"nodeCount"` 26 | Preimages map[common.Hash][]byte `json:"preimages"` 27 | } 28 | 29 | func TrieToJson(root common.Hash, step int) []byte { 30 | b, err := json.Marshal(Jtree{Preimages: Preimages, Step: step, Root: root}) 31 | check(err) 32 | return b 33 | } 34 | 35 | func TrieToJsonWithNodeID(root common.Hash, step int, nodeID int, nodeCount int) []byte { 36 | b, err := json.Marshal(Jtree{Preimages: Preimages, Step: step, NodeID: nodeID, NodeCount: nodeCount, Root: root}) 37 | check(err) 38 | return b 39 | } 40 | 41 | func TrieFromJson(dat []byte) (common.Hash, int) { 42 | var j Jtree 43 | err := json.Unmarshal(dat, &j) 44 | check(err) 45 | Preimages = j.Preimages 46 | return j.Root, j.Step 47 | } 48 | 49 | // TODO: this is copied from the oracle 50 | func (kw PreimageKeyValueWriter) Put(key []byte, value []byte) error { 51 | hash := crypto.Keccak256Hash(value) 52 | if hash != common.BytesToHash(key) { 53 | panic("bad preimage value write") 54 | } 55 | Preimages[hash] = common.CopyBytes(value) 56 | return nil 57 | } 58 | 59 | func (kw PreimageKeyValueWriter) Delete(key []byte) error { 60 | delete(Preimages, common.BytesToHash(key)) 61 | return nil 62 | } 63 | 64 | func ParseNodeInternal(elems []byte, depth int, callback func(common.Hash) []byte) { 65 | sprefix := strings.Repeat(" ", depth) 66 | c, _ := rlp.CountValues(elems) 67 | fmt.Println(sprefix, "parsing", depth, "elements", c) 68 | rest := elems 69 | for i := 0; i < c; i++ { 70 | kind, val, lrest, err := rlp.Split(rest) 71 | rest = lrest 72 | check(err) 73 | if len(val) > 0 { 74 | fmt.Println(sprefix, i, kind, val, len(val)) 75 | } 76 | if len(val) == 32 { 77 | hh := common.BytesToHash(val) 78 | //fmt.Println(sprefix, "node found with len", len(Preimages[hh])) 79 | ParseNode(hh, depth+1, callback) 80 | } 81 | if kind == rlp.List && len(val) > 0 && len(val) < 32 { 82 | ParseNodeInternal(val, depth+1, callback) 83 | } 84 | } 85 | } 86 | 87 | // full nodes / BRANCH_NODE have 17 values, each a hash 88 | // LEAF or EXTENSION nodes have 2 values, a path and value 89 | func ParseNode(node common.Hash, depth int, callback func(common.Hash) []byte) { 90 | if depth > 4 { 91 | return 92 | } 93 | buf := callback(node) 94 | //fmt.Println("callback", node, len(buf), hex.EncodeToString(buf)) 95 | elems, _, err := rlp.SplitList(buf) 96 | check(err) 97 | ParseNodeInternal(elems, depth, callback) 98 | } 99 | 100 | func RamFromTrie(root common.Hash) map[uint32](uint32) { 101 | ram := make(map[uint32](uint32)) 102 | 103 | // load into oracle 104 | pp := oracle.Preimages() 105 | for k, v := range Preimages { 106 | pp[k] = v 107 | } 108 | 109 | triedb := trie.Database{Root: root} 110 | tt, err := trie.New(root, &triedb) 111 | check(err) 112 | tni := tt.NodeIterator([]byte{}) 113 | for tni.Next(true) { 114 | if tni.Leaf() { 115 | tk := binary.BigEndian.Uint32(tni.LeafKey()) 116 | tv := binary.BigEndian.Uint32(tni.LeafBlob()) 117 | ram[tk*4] = tv 118 | } 119 | } 120 | return ram 121 | } 122 | 123 | func RamToTrie(ram map[uint32](uint32)) common.Hash { 124 | mt := trie.NewStackTrie(PreimageKeyValueWriter{}) 125 | 126 | sram := make([]uint64, len(ram)) 127 | 128 | i := 0 129 | for k, v := range ram { 130 | sram[i] = (uint64(k) << 32) | uint64(v) 131 | i += 1 132 | } 133 | sort.Slice(sram, func(i, j int) bool { return sram[i] < sram[j] }) 134 | 135 | for _, kv := range sram { 136 | k, v := uint32(kv>>32), uint32(kv) 137 | k >>= 2 138 | //fmt.Printf("insert %x = %x\n", k, v) 139 | tk := make([]byte, 4) 140 | tv := make([]byte, 4) 141 | binary.BigEndian.PutUint32(tk, k) 142 | binary.BigEndian.PutUint32(tv, v) 143 | mt.Update(tk, tv) 144 | } 145 | mt.Commit() 146 | /*fmt.Println("ram hash", mt.Hash()) 147 | fmt.Println("hash count", len(Preimages)) 148 | parseNode(mt.Hash(), 0)*/ 149 | return mt.Hash() 150 | } 151 | -------------------------------------------------------------------------------- /mlvm/vm/utils.go: -------------------------------------------------------------------------------- 1 | package vm 2 | 3 | import ( 4 | "encoding/binary" 5 | "io/ioutil" 6 | "time" 7 | ) 8 | 9 | var ministart time.Time 10 | 11 | func ZeroRegisters(ram map[uint32](uint32)) { 12 | for i := uint32(0xC0000000); i < 0xC0000000+36*4; i += 4 { 13 | WriteRam(ram, i, 0) 14 | } 15 | } 16 | 17 | func LoadData(dat []byte, ram map[uint32](uint32), base uint32) { 18 | for i := 0; i < len(dat); i += 4 { 19 | value := binary.BigEndian.Uint32(dat[i : i+4]) 20 | if value != 0 { 21 | ram[base+uint32(i)] = value 22 | } 23 | } 24 | } 25 | 26 | func LoadMappedFile(fn string, ram map[uint32](uint32), base uint32) { 27 | dat, err := ioutil.ReadFile(fn) 28 | check(err) 29 | LoadData(dat, ram, base) 30 | } -------------------------------------------------------------------------------- /mlvm/vm/vm.go: -------------------------------------------------------------------------------- 1 | package vm 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "errors" 7 | "flag" 8 | "fmt" 9 | "io/ioutil" 10 | "os" 11 | "strconv" 12 | 13 | uc "github.com/unicorn-engine/unicorn/bindings/go/unicorn" 14 | ) 15 | 16 | func WriteCheckpoint(ram map[uint32](uint32), fn string, step int) { 17 | trieroot := RamToTrie(ram) 18 | dat := TrieToJson(trieroot, step) 19 | fmt.Printf("writing %s len %d with root %s\n", fn, len(dat), trieroot) 20 | ioutil.WriteFile(fn, dat, 0644) 21 | } 22 | 23 | func WriteCheckpointWithNodeID(ram map[uint32](uint32), fn string, step int, nodeID int, nodeCount int) { 24 | trieroot := RamToTrie(ram) 25 | dat := TrieToJsonWithNodeID(trieroot, step, nodeID, nodeCount) 26 | fmt.Printf("writing %s len %d with root %s\n", fn, len(dat), trieroot) 27 | ioutil.WriteFile(fn, dat, 0644) 28 | } 29 | 30 | // memory layout in MIPS 31 | const ( 32 | INPUT_ADDR = 0x31000000 33 | OUTPUT_ADDR = 0x32000000 34 | MODEL_ADDR = 0x33000000 35 | MAGIC_ADDR = 0x30000800 36 | ) 37 | 38 | const ( 39 | MIPS_PROGRAM = "../../mlgo/ml_mips/ml_mips.bin" 40 | ) 41 | 42 | const ( 43 | READ_FROM_BIDENDIAN = true 44 | OUTPUT_TO_BIDENDIAN = true 45 | ) 46 | 47 | func IntToBytes(n int) []byte { 48 | x := int32(n) 49 | bytesBuffer := bytes.NewBuffer([]byte{}) 50 | if READ_FROM_BIDENDIAN{ 51 | binary.Write(bytesBuffer, binary.BigEndian, x) 52 | } else { 53 | binary.Write(bytesBuffer, binary.LittleEndian, x) 54 | } 55 | 56 | return bytesBuffer.Bytes() 57 | } 58 | 59 | func LoadModel(mu uc.Unicorn, file string, ram map[uint32](uint32)) { 60 | modelBytes, err := ioutil.ReadFile(file) 61 | if err != nil { 62 | fmt.Println(err) 63 | return 64 | } 65 | modelSize := len(modelBytes) 66 | fmt.Println("modelSize: ", modelSize) 67 | rawSize := IntToBytes(modelSize) 68 | fmt.Println("rawSize: ", rawSize) 69 | LoadBytesToUnicorn(mu, rawSize, ram, MODEL_ADDR) 70 | LoadBytesToUnicorn(mu, modelBytes, ram, MODEL_ADDR + 4) 71 | } 72 | 73 | func LoadInputData(mu uc.Unicorn, file string, ram map[uint32](uint32)) error { 74 | // load a random test digit 75 | buf, err := ioutil.ReadFile(file) 76 | if err != nil { 77 | fmt.Println(err) 78 | return err 79 | } 80 | if len(buf) >= 10 * 1024 * 1024 { 81 | fmt.Println("data too large") 82 | return errors.New("data too large") 83 | } 84 | //buf is the data 85 | inputSize := len(buf) 86 | LoadBytesToUnicorn(mu, IntToBytes(inputSize), ram, INPUT_ADDR) 87 | LoadBytesToUnicorn(mu, buf, ram, INPUT_ADDR + 4) 88 | 89 | return nil 90 | } 91 | 92 | type Params struct { 93 | Target int 94 | ProgramPath string 95 | ModelPath string 96 | InputPath string 97 | Basedir string 98 | OutputGolden bool 99 | 100 | CurLayer int 101 | LastLayer bool 102 | ModelName string 103 | NodeID int 104 | 105 | MIPSVMCompatible bool 106 | } 107 | 108 | func ParseParams() *Params { 109 | var target int 110 | var programPath string 111 | var modelPath string 112 | var inputPath string 113 | var basedir string 114 | var outputGolden bool 115 | 116 | var curLayer int 117 | var lastLayer bool 118 | var modelName string 119 | var nodeID int 120 | 121 | var mipsVMCompatible bool 122 | 123 | defaultBasedir := os.Getenv("BASEDIR") 124 | if len(defaultBasedir) == 0 { 125 | defaultBasedir = "/tmp/cannon" 126 | } 127 | flag.StringVar(&basedir, "basedir", defaultBasedir, "Directory to read inputs, write outputs, and cache preimage oracle data.") 128 | flag.IntVar(&target, "target", -1, "Target number of instructions to execute in the trace. If < 0 will execute until termination") 129 | flag.StringVar(&programPath, "program", MIPS_PROGRAM, "Path to binary file containing the program to run") 130 | flag.StringVar(&modelPath, "model", "", "Path to binary file containing the AI model") 131 | flag.StringVar(&inputPath, "data", "", "Path to binary file containing the input of AI model") 132 | flag.BoolVar(&outputGolden, "outputGolden", false, "Do not read any inputs and instead produce a snapshot of the state prior to execution. Written to /golden.json") 133 | 134 | flag.BoolVar(&lastLayer, "lastLayer", false, "In the lastLayer, we run computation in VM") 135 | flag.IntVar(&curLayer, "curLayer", 0, "The current layer") 136 | flag.StringVar(&modelName, "modelName", "MNIST", "run MNIST or LLAMA") 137 | flag.IntVar(&nodeID, "nodeID", 0, "The current nodeID") 138 | 139 | flag.BoolVar(&mipsVMCompatible, "mipsVMCompatible", false, "compatible for MIPS VM") 140 | flag.Parse() 141 | 142 | params := &Params{ 143 | Target: target, 144 | ProgramPath: programPath, 145 | ModelPath: modelPath, 146 | InputPath: inputPath, 147 | Basedir: basedir, 148 | OutputGolden: outputGolden, 149 | CurLayer: curLayer, 150 | LastLayer: lastLayer, 151 | ModelName: modelName, 152 | NodeID: nodeID, 153 | MIPSVMCompatible: mipsVMCompatible, 154 | } 155 | 156 | return params 157 | } 158 | 159 | func Run() { 160 | params := ParseParams() 161 | RunWithParams(params) 162 | } 163 | 164 | func RunWithParams(params *Params) { 165 | 166 | target := params.Target 167 | programPath := params.ProgramPath 168 | modelPath := params.ModelPath 169 | inputPath := params.InputPath 170 | basedir := params.Basedir 171 | outputGolden := params.OutputGolden 172 | // curLayer := params.CurLayer 173 | lastLayer := params.LastLayer 174 | modelName := params.ModelName 175 | nodeID := params.NodeID 176 | 177 | if params.MIPSVMCompatible { 178 | MIPSRunCompatible(basedir, target, programPath, modelPath, inputPath, outputGolden) 179 | return 180 | } 181 | 182 | if !lastLayer { 183 | id := target 184 | nodeFile, nodeCount, err := LayerRun(basedir + "/data", id, modelName) 185 | if err != nil { 186 | fmt.Println("layer run error: ", err) 187 | return 188 | } 189 | MIPSRun(basedir + "/checkpoint", 0, id, programPath, nodeFile, true, nodeCount) 190 | } else { 191 | // the lastLayer 192 | MIPSRun(basedir + "/checkpoint", target, nodeID, programPath, inputPath, outputGolden, 0) 193 | } 194 | 195 | 196 | // step 2 (optional), validate each 1 million chunk in EVM 197 | 198 | // step 3 (super optional) validate each 1 million chunk on chain 199 | 200 | //RunWithRam(ram, steps, debug, nil) 201 | 202 | } 203 | 204 | func LayerRun(basedir string, nodeID int, modelName string) (string, int, error) { 205 | var envBytes []byte 206 | var err error 207 | var nodeCount int 208 | 209 | if modelName == "MNIST" { 210 | envBytes, nodeCount, err = MNIST(nodeID) 211 | } else { // if modelName == "LLAMA" 212 | envBytes, nodeCount, err = LLAMA(nodeID) 213 | } 214 | 215 | if err != nil { 216 | fmt.Println("Layer run error: ", err) 217 | return "", nodeCount, err 218 | } 219 | 220 | fileName := fmt.Sprintf("%s/node_%d", basedir, nodeID) 221 | err = saveDataToFile(envBytes, fileName) 222 | 223 | if err != nil { 224 | fmt.Println("Save data error: ", err) 225 | return fileName, nodeCount, err 226 | } 227 | 228 | return fileName, nodeCount, nil 229 | } 230 | 231 | func saveDataToFile(data []byte, filename string) error { 232 | fout, err := os.Create(filename) 233 | if err != nil { 234 | fmt.Println(err) 235 | return err 236 | } 237 | defer fout.Close() 238 | _, err = fout.Write(data) 239 | if err != nil { 240 | fmt.Println(err) 241 | return err 242 | } 243 | return nil 244 | } 245 | 246 | func MIPSRun(basedir string, target int, nodeID int, programPath string, inputPath string, outputGolden bool, nodeCount int) { 247 | regfault := -1 248 | regfault_str, regfault_valid := os.LookupEnv("REGFAULT") 249 | if regfault_valid { 250 | regfault, _ = strconv.Atoi(regfault_str) 251 | } 252 | 253 | // step 1, generate the checkpoints every million steps using unicorn 254 | ram := make(map[uint32](uint32)) 255 | 256 | lastStep := 1 257 | reachFinalState := true // if the target >= total step, the targt will not be saved 258 | 259 | mu := GetHookedUnicorn(basedir, ram, func(step int, mu uc.Unicorn, ram map[uint32](uint32)) { 260 | if step == regfault { 261 | fmt.Printf("regfault at step %d\n", step) 262 | mu.RegWrite(uc.MIPS_REG_V0, 0xbabababa) 263 | } 264 | if step == target { 265 | reachFinalState = false 266 | SyncRegs(mu, ram) 267 | fn := fmt.Sprintf("%s/checkpoint_%d_%d.json", basedir, nodeID, step) 268 | WriteCheckpointWithNodeID(ram, fn, step, nodeID, nodeCount) 269 | if step == target { 270 | // done 271 | mu.RegWrite(uc.MIPS_REG_PC, 0x5ead0004) 272 | } 273 | } 274 | lastStep = step + 1 275 | }) 276 | 277 | ZeroRegisters(ram) 278 | // not ready for golden yet 279 | LoadMappedFileUnicorn(mu, programPath, ram, 0) 280 | // load input 281 | if inputPath != "" { 282 | LoadInputData(mu, inputPath, ram) 283 | } 284 | 285 | if outputGolden { 286 | WriteCheckpointWithNodeID(ram, fmt.Sprintf("%s/%d_golden.json", basedir, nodeID), -1, nodeID, nodeCount) 287 | fmt.Println("Writing golden snapshot and exiting early without execution") 288 | return 289 | } 290 | 291 | // do not need if we just run pure computation task 292 | // LoadMappedFileUnicorn(mu, fmt.Sprintf("%s/input", basedir), ram, 0x30000000) 293 | 294 | mu.Start(0, 0x5ead0004) 295 | 296 | 297 | if reachFinalState { 298 | fmt.Printf("reach the final state, total step: %d, target: %d\n", lastStep, target) 299 | WriteCheckpointWithNodeID(ram, fmt.Sprintf("%s/checkpoint_%d_%d.json", basedir, nodeID, lastStep), lastStep, nodeID, nodeCount) 300 | } 301 | 302 | if target == -1 { 303 | 304 | fmt.Println("lastStep: ", lastStep) 305 | WriteCheckpointWithNodeID(ram, fmt.Sprintf("%s/checkpoint_%d_final.json", basedir, nodeID), lastStep, nodeID, nodeCount) 306 | 307 | } 308 | } 309 | 310 | func MIPSRunCompatible(basedir string, target int, programPath string, modelPath string, inputPath string, outputGolden bool) { 311 | regfault := -1 312 | regfault_str, regfault_valid := os.LookupEnv("REGFAULT") 313 | if regfault_valid { 314 | regfault, _ = strconv.Atoi(regfault_str) 315 | } 316 | 317 | // step 1, generate the checkpoints every million steps using unicorn 318 | ram := make(map[uint32](uint32)) 319 | 320 | lastStep := 1 321 | reachFinalState := true // if the target >= total step, the targt will not be saved 322 | 323 | mu := GetHookedUnicorn(basedir, ram, func(step int, mu uc.Unicorn, ram map[uint32](uint32)) { 324 | if step == regfault { 325 | fmt.Printf("regfault at step %d\n", step) 326 | mu.RegWrite(uc.MIPS_REG_V0, 0xbabababa) 327 | } 328 | if step == target { 329 | reachFinalState = false 330 | SyncRegs(mu, ram) 331 | fn := fmt.Sprintf("%s/checkpoint_%d.json", basedir, step) 332 | WriteCheckpoint(ram, fn, step) 333 | if step == target { 334 | // done 335 | mu.RegWrite(uc.MIPS_REG_PC, 0x5ead0004) 336 | } 337 | } 338 | lastStep = step + 1 339 | }) 340 | 341 | ZeroRegisters(ram) 342 | // not ready for golden yet 343 | LoadMappedFileUnicorn(mu, programPath, ram, 0) 344 | // load input 345 | if inputPath != "" { 346 | LoadInputData(mu, inputPath, ram) 347 | } 348 | LoadModel(mu, modelPath, ram) 349 | 350 | 351 | if outputGolden { 352 | WriteCheckpoint(ram, fmt.Sprintf("%s/golden.json", basedir), -1) 353 | fmt.Println("Writing golden snapshot and exiting early without execution") 354 | return 355 | } 356 | 357 | // do not need if we just run pure computation task 358 | // LoadMappedFileUnicorn(mu, fmt.Sprintf("%s/input", basedir), ram, 0x30000000) 359 | 360 | SyncRegs(mu, ram) 361 | mu.Start(0, 0x5ead0004) 362 | SyncRegs(mu, ram) 363 | 364 | if reachFinalState { 365 | fmt.Printf("reach the final state, total step: %d, target: %d\n", lastStep, target) 366 | WriteCheckpoint(ram, fmt.Sprintf("%s/checkpoint_%d.json", basedir, lastStep), lastStep) 367 | } 368 | 369 | if target == -1 { 370 | 371 | fmt.Println("lastStep: ", lastStep) 372 | WriteCheckpoint(ram, fmt.Sprintf("%s/checkpoint_final.json", basedir), lastStep) 373 | fmt.Printf("PC: %x\n", ram[0xC0000080]) 374 | } 375 | } -------------------------------------------------------------------------------- /mlvm/vm/vm_test.go: -------------------------------------------------------------------------------- 1 | package vm 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/ethereum/go-ethereum/common" 7 | ) 8 | 9 | func initTest() { 10 | Preimages = make(map[common.Hash][]byte) 11 | steps = 0 12 | heap_start = 0 13 | } 14 | 15 | func TestVM(t *testing.T){ 16 | initTest() 17 | params := &Params{ 18 | Target: 2, 19 | ProgramPath: MIPS_PROGRAM, 20 | Basedir: "/tmp/cannon", 21 | ModelName: "MNIST", 22 | LastLayer: false, 23 | NodeID: 0, 24 | } 25 | RunWithParams(params) 26 | } 27 | 28 | func TestVM1(t *testing.T){ 29 | initTest() 30 | params := &Params{ 31 | Target: 0, 32 | ProgramPath: MIPS_PROGRAM, 33 | Basedir: "/tmp/cannon", 34 | ModelName: "LLAMA", 35 | LastLayer: false, 36 | NodeID: 0, 37 | } 38 | RunWithParams(params) 39 | } 40 | 41 | func TestVM2(t *testing.T){ 42 | initTest() 43 | params := &Params{ 44 | Target: -1, 45 | ProgramPath: MIPS_PROGRAM, 46 | Basedir: "/tmp/cannon", 47 | ModelName: "MNIST", 48 | LastLayer: true, 49 | InputPath: "/tmp/cannon/data/node_2", 50 | NodeID: 2, 51 | } 52 | RunWithParams(params) 53 | } 54 | 55 | func TestVM3(t *testing.T){ 56 | initTest() 57 | params := &Params{ 58 | Target: -1, 59 | ProgramPath: "../../mlgo/examples/mnist_mips/mlgo.bin", 60 | Basedir: "/tmp/cannon", 61 | ModelPath: "../../mlgo/examples/mnist/models/mnist/ggml-model-small-f32-big-endian.bin", 62 | InputPath: "../../mlgo/examples/mnist/models/mnist/input_7", 63 | MIPSVMCompatible: true, 64 | } 65 | RunWithParams(params) 66 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "@nomiclabs/hardhat-ethers": "^2.0.5", 4 | "@types/node": "^17.0.31", 5 | "chai": "^4.3.6", 6 | "ethereum-waffle": "^3.4.4", 7 | "ethers": "5.6.1", 8 | "hardhat-gas-reporter": "^1.0.8", 9 | "rlp": "^3.0.0", 10 | "typescript": "^4.6.4" 11 | }, 12 | "devDependencies": { 13 | "@nomicfoundation/hardhat-toolbox": "^2.0.2", 14 | "hardhat": "^2.18.2" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /scripts/assert.js: -------------------------------------------------------------------------------- 1 | const { deployed, getTrieNodesForCall, getTrieAtStep } = require("../scripts/lib") 2 | 3 | async function main() { 4 | let [c, m, mm] = await deployed() 5 | 6 | const challengeId = parseInt(process.env.ID) 7 | const isChallenger = process.env.CHALLENGER == "1" 8 | 9 | let step = (await c.getStepNumber(challengeId)).toNumber() 10 | console.log("searching step", step) 11 | 12 | if (await c.isSearching(challengeId)) { 13 | console.log("search is NOT done") 14 | return 15 | } 16 | 17 | let cdat 18 | if (isChallenger) { 19 | // challenger declare victory 20 | cdat = c.interface.encodeFunctionData("confirmStateTransition", [challengeId]) 21 | } else { 22 | // defender declare victory 23 | // note: not always possible 24 | cdat = c.interface.encodeFunctionData("denyStateTransition", [challengeId]) 25 | } 26 | 27 | let startTrie = getTrieAtStep(step) 28 | let finalTrie = getTrieAtStep(step+1) 29 | let preimages = Object.assign({}, startTrie['preimages'], finalTrie['preimages']); 30 | 31 | let nodes = await getTrieNodesForCall(c, c.address, cdat, preimages) 32 | for (n of nodes) { 33 | await mm.AddTrieNode(n) 34 | } 35 | 36 | let ret 37 | if (isChallenger) { 38 | ret = await c.confirmStateTransition(challengeId) 39 | } else { 40 | ret = await c.denyStateTransition(challengeId) 41 | } 42 | 43 | let receipt = await ret.wait() 44 | console.log(receipt.events.map((x) => x.event)) 45 | } 46 | 47 | main() 48 | .then(() => process.exit(0)) 49 | .catch((error) => { 50 | console.error(error); 51 | process.exit(1); 52 | }); 53 | -------------------------------------------------------------------------------- /scripts/challenge.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs") 2 | const { basedir, deployed, getBlockRlp, getTrieNodesForCall } = require("../scripts/lib") 3 | 4 | async function main() { 5 | let [c, m, mm] = await deployed() 6 | 7 | console.log(c.address, m.address, mm.address) 8 | console.log("basedir: ", basedir) 9 | 10 | // TODO: move this to lib, it's shared with the test 11 | let startTrie = JSON.parse(fs.readFileSync(basedir+"/golden.json")) 12 | let finalTrie = JSON.parse(fs.readFileSync(basedir+"/checkpoint_final.json")) 13 | let preimages = Object.assign({}, startTrie['preimages'], finalTrie['preimages']); 14 | const finalSystemState = finalTrie['root'] 15 | 16 | let args = [finalSystemState, finalTrie['step']] 17 | let cdat = c.interface.encodeFunctionData("initiatePureComputationChallenge", args) 18 | let nodes = await getTrieNodesForCall(c, c.address, cdat, preimages) 19 | console.log("cdat: ", cdat) 20 | 21 | // run "on chain" 22 | for (n of nodes) { 23 | await mm.AddTrieNode(n) 24 | } 25 | // TODO: Setting the gas limit explicitly here shouldn't be necessary, for some 26 | // weird reason (to be investigated), it is for L2. 27 | // let ret = await c.initiateChallenge(...args) 28 | let ret = await c.initiatePureComputationChallenge(...args) 29 | let receipt = await ret.wait() 30 | // ChallengeCreated event 31 | let challengeId = receipt.events[0].args['challengeId'].toNumber() 32 | console.log("new challenge with id", challengeId) 33 | } 34 | 35 | main() 36 | .then(() => process.exit(0)) 37 | .catch((error) => { 38 | console.error(error); 39 | process.exit(1); 40 | }); 41 | -------------------------------------------------------------------------------- /scripts/deploy.js: -------------------------------------------------------------------------------- 1 | const { deploy } = require("../scripts/lib") 2 | const fs = require("fs") 3 | 4 | async function main() { 5 | let [c, m, mm] = await deploy() 6 | let json = { 7 | "Challenge": c.address, 8 | "MIPS": m.address, 9 | "MIPSMemory": mm.address, 10 | } 11 | console.log("deployed", json) 12 | fs.writeFileSync("/tmp/cannon/deployed.json", JSON.stringify(json)) 13 | } 14 | 15 | main() 16 | .then(() => process.exit(0)) 17 | .catch((error) => { 18 | console.error(error); 19 | process.exit(1); 20 | }); 21 | -------------------------------------------------------------------------------- /scripts/lib.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs") 2 | const rlp = require('rlp') 3 | const child_process = require("child_process") 4 | 5 | const basedir = process.env.BASEDIR == undefined ? "/tmp/cannon" : process.env.BASEDIR 6 | const programPath = process.env.PROGRAM_PATH 7 | const modelPath = process.env.MODEL_PATH 8 | const dataPath = process.env.DATA_PATH 9 | 10 | async function deploy() { 11 | const MIPS = await ethers.getContractFactory("MIPS") 12 | const m = await MIPS.deploy() 13 | const mm = await ethers.getContractAt("MIPSMemory", await m.m()) 14 | 15 | let startTrie = JSON.parse(fs.readFileSync(basedir+"/golden.json")) 16 | let goldenRoot = startTrie["root"] 17 | console.log("goldenRoot is", goldenRoot) 18 | 19 | const Challenge = await ethers.getContractFactory("Challenge") 20 | const c = await Challenge.deploy(m.address, goldenRoot) 21 | 22 | return [c,m,mm] 23 | } 24 | 25 | function getBlockRlp(block) { 26 | let dat = [ 27 | block['parentHash'], 28 | block['sha3Uncles'], 29 | block['miner'], 30 | block['stateRoot'], 31 | block['transactionsRoot'], 32 | block['receiptsRoot'], 33 | block['logsBloom'], 34 | block['difficulty'], 35 | block['number'], 36 | block['gasLimit'], 37 | block['gasUsed'], 38 | block['timestamp'], 39 | block['extraData'], 40 | block['mixHash'], 41 | block['nonce'], 42 | ]; 43 | // post london 44 | if (block['baseFeePerGas'] !== undefined) { 45 | dat.push(block['baseFeePerGas']) 46 | } 47 | dat = dat.map(x => (x == "0x0") ? "0x" : x) 48 | //console.log(dat) 49 | let rdat = rlp.encode(dat) 50 | if (ethers.utils.keccak256(rdat) != block['hash']) { 51 | throw "block hash doesn't match" 52 | } 53 | return rdat 54 | } 55 | 56 | async function deployed() { 57 | let addresses = JSON.parse(fs.readFileSync(basedir+"/deployed.json")) 58 | const c = await ethers.getContractAt("Challenge", addresses["Challenge"]) 59 | const m = await ethers.getContractAt("MIPS", addresses["MIPS"]) 60 | const mm = await ethers.getContractAt("MIPSMemory", addresses["MIPSMemory"]) 61 | return [c,m,mm] 62 | } 63 | 64 | class MissingHashError extends Error { 65 | constructor(hash, offset) { 66 | super("hash is missing") 67 | this.hash = hash 68 | this.offset = offset 69 | } 70 | } 71 | 72 | async function getTrieNodesForCall(c, caddress, cdat, preimages) { 73 | let nodes = [] 74 | while (1) { 75 | try { 76 | // TODO: make this eth call? 77 | // needs something like initiateChallengeWithTrieNodesj 78 | let calldata = c.interface.encodeFunctionData("callWithTrieNodes", [caddress, cdat, nodes]) 79 | ret = await ethers.provider.call({ 80 | to:c.address, 81 | data:calldata 82 | }); 83 | break 84 | } catch(e) { 85 | let missing = e.toString().split("'")[1] 86 | if (missing == undefined) { 87 | // other kind of error from HTTPProvider 88 | missing = e.error.message.toString().split("execution reverted: ")[1] 89 | } 90 | if (missing !== undefined && missing.length == 64) { 91 | console.log("requested node", missing) 92 | let node = preimages["0x"+missing] 93 | if (node === undefined) { 94 | throw("node not found") 95 | } 96 | const bin = Uint8Array.from(Buffer.from(node, 'base64').toString('binary'), c => c.charCodeAt(0)) 97 | nodes.push(bin) 98 | continue 99 | } else if (missing !== undefined && missing.length == 128) { 100 | let hash = missing.slice(0, 64) 101 | let offset = parseInt(missing.slice(64, 128), 16) 102 | console.log("requested hash oracle", hash, offset) 103 | throw new MissingHashError(hash, offset) 104 | } else { 105 | console.log(e) 106 | break 107 | } 108 | } 109 | } 110 | return nodes 111 | } 112 | 113 | function getTrieAtStep(step) { 114 | const fn = basedir+"/checkpoint_"+step.toString()+".json" 115 | 116 | if (!fs.existsSync(fn)) { 117 | // console.log("running mipsevm") 118 | console.log("running program: ", programPath) 119 | child_process.execSync("mlvm/mlvm --mipsVMCompatible" + " --target="+step.toString() + " --program="+programPath + " --model="+modelPath + " --data="+dataPath, {stdio: 'inherit'}) 120 | } 121 | 122 | return JSON.parse(fs.readFileSync(fn)) 123 | } 124 | 125 | 126 | async function writeMemory(mm, root, addr, data, bytes32=false) { 127 | if (bytes32) { 128 | ret = await mm.WriteBytes32WithReceipt(root, addr, data) 129 | } else { 130 | ret = await mm.WriteMemoryWithReceipt(root, addr, data) 131 | } 132 | const receipt = await ret.wait() 133 | for (l of receipt.logs) { 134 | if (l.topics[0] == "0x86b89b5c9818dbbf520dd979a5f250d357508fe11b9511d4a43fd9bc6aa1be70") { 135 | root = l.data 136 | } 137 | } 138 | console.log("new hash", root) 139 | return root 140 | } 141 | 142 | module.exports = { basedir, deploy, deployed, getTrieNodesForCall, getBlockRlp, getTrieAtStep, writeMemory, MissingHashError } 143 | -------------------------------------------------------------------------------- /scripts/respond.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs") 2 | const { deployed, getTrieNodesForCall, getTrieAtStep } = require("../scripts/lib") 3 | 4 | async function main() { 5 | let [c, m, mm] = await deployed() 6 | 7 | const challengeId = parseInt(process.env.ID) 8 | const isChallenger = process.env.CHALLENGER == "1" 9 | 10 | console.log("challengeId: ", challengeId) 11 | console.log("isChallenger: ", isChallenger) 12 | 13 | let step = (await c.getStepNumber(challengeId)).toNumber() 14 | console.log("searching step", step) 15 | 16 | if (!(await c.isSearching(challengeId))) { 17 | console.log("search is done") 18 | return 19 | } 20 | 21 | // see if it's proposed or not 22 | const proposed = await c.getProposedState(challengeId) 23 | const isProposing = proposed == "0x0000000000000000000000000000000000000000000000000000000000000000" 24 | if (isProposing != isChallenger) { 25 | console.log("bad challenger state") 26 | return 27 | } 28 | console.log("isProposing", isProposing) 29 | let thisTrie = getTrieAtStep(step) 30 | const root = thisTrie['root'] 31 | console.log("new root", root) 32 | 33 | let ret 34 | if (isProposing) { 35 | ret = await c.proposeState(challengeId, root) 36 | } else { 37 | ret = await c.respondState(challengeId, root) 38 | } 39 | let receipt = await ret.wait() 40 | console.log("done", receipt.blockNumber) 41 | } 42 | 43 | main() 44 | .then(() => process.exit(0)) 45 | .catch((error) => { 46 | console.error(error); 47 | process.exit(1); 48 | }); -------------------------------------------------------------------------------- /test/challenge_test.js: -------------------------------------------------------------------------------- 1 | const { expect } = require("chai") 2 | const fs = require("fs") 3 | const { deploy, getTrieNodesForCall } = require("../scripts/lib") 4 | 5 | // This test needs preimages to run correctly. 6 | // It is skipped when running `make test_contracts`, but can be run with `make test_challenge`. 7 | describe("Challenge contract", function () { 8 | if (!fs.existsSync("/tmp/cannon/golden.json")) { 9 | console.log("golden file doesn't exist, skipping test") 10 | return 11 | } 12 | 13 | beforeEach(async function () { 14 | [c, m, mm] = await deploy() 15 | }) 16 | it("challenge contract deploys", async function() { 17 | console.log("Challenge deployed at", c.address) 18 | }) 19 | it("initiate challenge", async function() { 20 | 21 | let startTrie = JSON.parse(fs.readFileSync("/tmp/cannon/golden.json")) 22 | let finalTrie = JSON.parse(fs.readFileSync("/tmp/cannon/checkpoint_final.json")) 23 | let preimages = Object.assign({}, startTrie['preimages'], finalTrie['preimages']); 24 | const finalSystemState = finalTrie['root'] 25 | 26 | let args = [finalSystemState, finalTrie['step']] 27 | let cdat = c.interface.encodeFunctionData("initiatePureComputationChallenge", args) 28 | // console.log("contract: ", c) 29 | // console.log("cdat: ", cdat) 30 | let nodes = await getTrieNodesForCall(c, c.address, cdat, preimages) 31 | // console.log("what?") 32 | // run "on chain" 33 | for (n of nodes) { 34 | await mm.AddTrieNode(n) 35 | } 36 | let ret = await c.initiatePureComputationChallenge(...args) 37 | let receipt = await ret.wait() 38 | // ChallengeCreated event 39 | let challengeId = receipt.events[0].args['challengeId'].toNumber() 40 | console.log("new challenge with id", challengeId) 41 | let challengerResults = await c.challengerResults() 42 | console.log(challengerResults) 43 | 44 | // the real issue here is from step 0->1 when we write the input hash 45 | // TODO: prove the challenger wrong? 46 | }).timeout(200_000) 47 | }) 48 | -------------------------------------------------------------------------------- /test/libkeccak.js: -------------------------------------------------------------------------------- 1 | const { keccak256 } = require("@ethersproject/keccak256"); 2 | const { expect } = require("chai"); 3 | 4 | describe("MIPSMemory contract", function () { 5 | beforeEach(async function () { 6 | const MIPSMemory = await ethers.getContractFactory("MIPSMemory"); 7 | mm = await MIPSMemory.deploy(); 8 | console.log("deployed at", mm.address); 9 | }) 10 | it("Keccak should work", async function () { 11 | await mm.AddLargePreimageInit(0); 12 | console.log("preimage initted"); 13 | 14 | // empty 15 | async function tl(n) { 16 | const test = new Uint8Array(n) 17 | for (var i = 0; i < n; i++) test[i] = 0x62; 18 | console.log("test size", n) 19 | expect((await mm.AddLargePreimageFinal(test))[0]).to.equal(keccak256(test)); 20 | } 21 | await tl(1) 22 | await tl(100) 23 | await tl(134) 24 | await tl(135) 25 | 26 | // block size is 136 27 | let dat = new Uint8Array(136) 28 | dat[0] = 0x61 29 | await mm.AddLargePreimageUpdate(dat); 30 | 31 | const hash = (await mm.AddLargePreimageFinal([]))[0]; 32 | console.log("preimage updated"); 33 | 34 | const realhash = keccak256(dat); 35 | console.log("comp hash is", hash); 36 | console.log("real hash is", realhash); 37 | expect(hash).to.equal(realhash); 38 | }); 39 | it("oracle save should work", async function () { 40 | await mm.AddLargePreimageInit(4) 41 | 42 | let dat = new TextEncoder("utf-8").encode("hello world") 43 | let dathash = keccak256(dat) 44 | const tst = await mm.AddLargePreimageFinal(dat) 45 | expect(tst[0]).to.equal(dathash) 46 | expect(tst[1]).to.equal(11) 47 | expect(tst[2]).to.equal(0x6f20776f) 48 | 49 | await mm.AddLargePreimageFinalSaved(dat) 50 | await mm.AddPreimage(dat, 0) 51 | 52 | let retl = await mm.GetPreimageLength(dathash) 53 | let ret = await mm.GetPreimage(dathash, 4) 54 | expect(retl).to.equal(11) 55 | expect(ret).to.equal(0x6f20776f) 56 | 57 | // other type 58 | retl = await mm.GetPreimageLength(dathash) 59 | ret = await mm.GetPreimage(dathash, 0) 60 | expect(retl).to.equal(11) 61 | expect(ret).to.equal(0x68656c6c) 62 | }) 63 | }); -------------------------------------------------------------------------------- /test/mips_test_execwtrie.js: -------------------------------------------------------------------------------- 1 | const { expect } = require("chai"); 2 | const { execPath } = require("process"); 3 | 4 | const trieAdd = {"root":"0x22ffce7c56d926c2d8d6337d8917fa0e1880e1869e189c15385ead63c6c45b93","preimages":{"0x044371dc86fb8c621bc84b69dce16de366de1126777250888b17416d0bd11279":"+FPGIIQ8EL//xiCENhD/8MYghDQRAAHGIIQ8CP//xiCENQj//cYghDQJAAPGIIQBCVAgxiCELUIAAcYghK4CAAjGIISuEQAExiCEA+AACICAgICAgA==","0x0fdfcc24b1b21d78ef2b7c6503eb9354677743685c2d00a14a8b502a177911b0":"+HGgL4Jb+u0gEWM4e9G4lO/GsyUEY/heVoGOAfiI04qPXfSgLCZprT7WBOLipiwJxxI0vy09rw9iPR+x0p/Xz1p3X5WgaNY/x30waJPsd6PWg76b094l8vUmmL6XB1XdUFy+xfWAgICAgICAgICAgICAgA==","0x11228d4f4a028a9088e6ec0aa6513e0d4731d9dc488e2af1957e46ba80624a69":"5oQAAAAAoARDcdyG+4xiG8hLadzhbeNm3hEmd3JQiIsXQW0L0RJ5","0x22ffce7c56d926c2d8d6337d8917fa0e1880e1869e189c15385ead63c6c45b93":"+FGgESKNT0oCipCI5uwKplE+DUcx2dxIjirxlX5GuoBiSmmAgKBv5gezlmGtxjQAs8Du76D93mAxExw5qWgAZjJQp1xmfICAgICAgICAgICAgIA=","0x2c2669ad3ed604e2e2a62c09c71234bf2d3daf0f623d1fb1d29fd7cf5a775f95":"+HHGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIRerQAAgA==","0x2f825bfaed201163387bd1b894efc6b3250463f85e56818e01f888d38a8f5df4":"+HHGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAgA==","0x68d63fc77d306893ec77a3d683be9bd3de25f2f52698be970755dd505cbec5f5":"6cYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAACAgICAgICAgICAgICA","0x6fe607b39661adc63400b3c0eeefa0fdde6031131c39a96800663250a75c667c":"5YMQAACgD9/MJLGyHXjvK3xlA+uTVGd3Q2hcLQChSotQKhd5EbA="}}; 5 | const trieOracle = {"root":"0x26595bd4f73f14273d9f43f0c5253eef964f4581d5e293cded54d23f1cf3db5f","step":-1,"preimages":{"0x0fdfcc24b1b21d78ef2b7c6503eb9354677743685c2d00a14a8b502a177911b0":"+HGgL4Jb+u0gEWM4e9G4lO/GsyUEY/heVoGOAfiI04qPXfSgLCZprT7WBOLipiwJxxI0vy09rw9iPR+x0p/Xz1p3X5WgaNY/x30waJPsd6PWg76b094l8vUmmL6XB1XdUFy+xfWAgICAgICAgICAgICAgA==","0x16e7f9821e0a3a2fc92d337f6d269c4c0ddb4d5c10a04b49d368643c9818ec1d":"5YMQAACgoLKCLk7kk3rBB2CtW7YQizz670HG0TNCobpfs/1MJqs=","0x26595bd4f73f14273d9f43f0c5253eef964f4581d5e293cded54d23f1cf3db5f":"+FGgFuf5gh4KOi/JLTN/bSacTA3bTVwQoEtJ02hkPJgY7B2AgKBv5gezlmGtxjQAs8Du76D93mAxExw5qWgAZjJQp1xmfICAgICAgICAgICAgIA=","0x2c2669ad3ed604e2e2a62c09c71234bf2d3daf0f623d1fb1d29fd7cf5a775f95":"+HHGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIRerQAAgA==","0x2f825bfaed201163387bd1b894efc6b3250463f85e56818e01f888d38a8f5df4":"+HHGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAgA==","0x65c024ed3b68b3f86a44f6af5179e4ad055ac046ac2e954355d476c611947b16":"+HHGIISuCAAQxiCEPAhCpcYghDUI7F/GIISuCAAUxiCEPAgDu8YghDUI+iXGIISuCAAYxiCEPAhMsMYghDUIH63GIISuCAAcxiCEJAIPtMYghAAAAAzGIIQ8ETEAxiCEjigAAMYghCQMAAvGIIQBDGgjgA==","0x68d63fc77d306893ec77a3d683be9bd3de25f2f52698be970755dd505cbec5f5":"6cYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAACAgICAgICAgICAgICA","0x6fe607b39661adc63400b3c0eeefa0fdde6031131c39a96800663250a75c667c":"5YMQAACgD9/MJLGyHXjvK3xlA+uTVGd3Q2hcLQChSotQKhd5EbA=","0x8cd6ed962850eebdb5bf360b496b5ab3425659a8ba3d115d5bb71055981a6bc2":"+F/GIIQtogABxiCEjigABMYghDwMaGXGIIQ1jGxsxiCEAQxoI8YghC2jAAHGIIQAQxAkxiCEPBC//8YghDYQ//DGIIQ0EQABxiCErgIACMYghK4RAATGIIQD4AAIgICAgA==","0xa0b2822e4ee4937ac10760ad5bb6108b3cfaef41c6d13342a1ba5fb3fd4c26ab":"+HGgt8fQkZe4faINYK98tz2Czvb1LZ/LaDPtw0E2aioQ4lagZcAk7Ttos/hqRPavUXnkrQVawEasLpVDVdR2xhGUexagjNbtlihQ7r21vzYLSWtas0JWWai6PRFdW7cQVZgaa8KAgICAgICAgICAgICAgA==","0xb7c7d09197b87da20d60af7cb73d82cef6f52d9fcb6833edc341366a2a10e256":"+HHGIIQ8EDAAxiCENhAQAMYghDwIRxfGIIQ1CDKFxiCErggAAMYghDwIqNfGIIQ1CDQexiCErggABMYghDwIXpfGIIQ1CC/GxiCErggACMYghDwIdyjGIIQ1CGOExiCErggADMYghDwI+ALGIIQ1CPjvgA=="}}; 6 | 7 | async function addPreimages(mm, trie) { 8 | for (k in trie['preimages']) { 9 | const bin = Uint8Array.from(Buffer.from(trie['preimages'][k], 'base64').toString('binary'), c => c.charCodeAt(0)) 10 | await mm.AddTrieNode(bin) 11 | } 12 | return trie['root'] 13 | } 14 | 15 | describe("MIPS contract", function () { 16 | beforeEach(async function () { 17 | const MIPS = await ethers.getContractFactory("MIPS") 18 | m = await MIPS.deploy() 19 | mm = await ethers.getContractAt("MIPSMemory", await m.m()) 20 | }) 21 | 22 | it("add should work", async function () { 23 | let root = await addPreimages(mm, trieAdd) 24 | 25 | console.log("start", root) 26 | for (let i = 0; i < 12; i++) { 27 | ret = await m.Step(root) 28 | const receipt = await ret.wait() 29 | for (l of receipt.logs) { 30 | if (l.topics[0] == "0x86b89b5c9818dbbf520dd979a5f250d357508fe11b9511d4a43fd9bc6aa1be70") { 31 | root = l.data 32 | } 33 | } 34 | console.log(i, root) 35 | } 36 | }).timeout(40_000); 37 | 38 | it("oracle should work", async function () { 39 | let root = await addPreimages(mm, trieOracle) 40 | 41 | // "hello world" is the oracle 42 | await mm.AddPreimage(Buffer.from("hello world"), 0) 43 | 44 | console.log("start", root) 45 | let pc = 0, out1, out2 46 | while (pc != 0x5ead0000) { 47 | ret = await m.Step(root) 48 | const receipt = await ret.wait() 49 | for (l of receipt.logs) { 50 | if (l.topics[0] == "0x86b89b5c9818dbbf520dd979a5f250d357508fe11b9511d4a43fd9bc6aa1be70") { 51 | root = l.data 52 | } 53 | } 54 | pc = await mm.ReadMemory(root, 0xc0000080) 55 | out1 = await mm.ReadMemory(root, 0xbffffff4) 56 | out2 = await mm.ReadMemory(root, 0xbffffff8) 57 | console.log(root, pc, out1, out2) 58 | } 59 | expect(out1).to.equal(1) 60 | expect(out2).to.equal(1) 61 | }).timeout(200_000); 62 | 63 | }); 64 | -------------------------------------------------------------------------------- /test/mips_test_execwtrie_dynamic.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs") 2 | const { expect } = require("chai") 3 | const { deploy, getTrieNodesForCall, MissingHashError } = require("../scripts/lib") 4 | 5 | const trieAdd = {"root":"0x22ffce7c56d926c2d8d6337d8917fa0e1880e1869e189c15385ead63c6c45b93","preimages":{"0x044371dc86fb8c621bc84b69dce16de366de1126777250888b17416d0bd11279":"+FPGIIQ8EL//xiCENhD/8MYghDQRAAHGIIQ8CP//xiCENQj//cYghDQJAAPGIIQBCVAgxiCELUIAAcYghK4CAAjGIISuEQAExiCEA+AACICAgICAgA==","0x0fdfcc24b1b21d78ef2b7c6503eb9354677743685c2d00a14a8b502a177911b0":"+HGgL4Jb+u0gEWM4e9G4lO/GsyUEY/heVoGOAfiI04qPXfSgLCZprT7WBOLipiwJxxI0vy09rw9iPR+x0p/Xz1p3X5WgaNY/x30waJPsd6PWg76b094l8vUmmL6XB1XdUFy+xfWAgICAgICAgICAgICAgA==","0x11228d4f4a028a9088e6ec0aa6513e0d4731d9dc488e2af1957e46ba80624a69":"5oQAAAAAoARDcdyG+4xiG8hLadzhbeNm3hEmd3JQiIsXQW0L0RJ5","0x22ffce7c56d926c2d8d6337d8917fa0e1880e1869e189c15385ead63c6c45b93":"+FGgESKNT0oCipCI5uwKplE+DUcx2dxIjirxlX5GuoBiSmmAgKBv5gezlmGtxjQAs8Du76D93mAxExw5qWgAZjJQp1xmfICAgICAgICAgICAgIA=","0x2c2669ad3ed604e2e2a62c09c71234bf2d3daf0f623d1fb1d29fd7cf5a775f95":"+HHGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIRerQAAgA==","0x2f825bfaed201163387bd1b894efc6b3250463f85e56818e01f888d38a8f5df4":"+HHGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAgA==","0x68d63fc77d306893ec77a3d683be9bd3de25f2f52698be970755dd505cbec5f5":"6cYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAACAgICAgICAgICAgICA","0x6fe607b39661adc63400b3c0eeefa0fdde6031131c39a96800663250a75c667c":"5YMQAACgD9/MJLGyHXjvK3xlA+uTVGd3Q2hcLQChSotQKhd5EbA="}}; 6 | const trieOracle = {"root":"0x26595bd4f73f14273d9f43f0c5253eef964f4581d5e293cded54d23f1cf3db5f","step":-1,"preimages":{"0x0fdfcc24b1b21d78ef2b7c6503eb9354677743685c2d00a14a8b502a177911b0":"+HGgL4Jb+u0gEWM4e9G4lO/GsyUEY/heVoGOAfiI04qPXfSgLCZprT7WBOLipiwJxxI0vy09rw9iPR+x0p/Xz1p3X5WgaNY/x30waJPsd6PWg76b094l8vUmmL6XB1XdUFy+xfWAgICAgICAgICAgICAgA==","0x16e7f9821e0a3a2fc92d337f6d269c4c0ddb4d5c10a04b49d368643c9818ec1d":"5YMQAACgoLKCLk7kk3rBB2CtW7YQizz670HG0TNCobpfs/1MJqs=","0x26595bd4f73f14273d9f43f0c5253eef964f4581d5e293cded54d23f1cf3db5f":"+FGgFuf5gh4KOi/JLTN/bSacTA3bTVwQoEtJ02hkPJgY7B2AgKBv5gezlmGtxjQAs8Du76D93mAxExw5qWgAZjJQp1xmfICAgICAgICAgICAgIA=","0x2c2669ad3ed604e2e2a62c09c71234bf2d3daf0f623d1fb1d29fd7cf5a775f95":"+HHGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIRerQAAgA==","0x2f825bfaed201163387bd1b894efc6b3250463f85e56818e01f888d38a8f5df4":"+HHGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAADGIIQAAAAAgA==","0x65c024ed3b68b3f86a44f6af5179e4ad055ac046ac2e954355d476c611947b16":"+HHGIISuCAAQxiCEPAhCpcYghDUI7F/GIISuCAAUxiCEPAgDu8YghDUI+iXGIISuCAAYxiCEPAhMsMYghDUIH63GIISuCAAcxiCEJAIPtMYghAAAAAzGIIQ8ETEAxiCEjigAAMYghCQMAAvGIIQBDGgjgA==","0x68d63fc77d306893ec77a3d683be9bd3de25f2f52698be970755dd505cbec5f5":"6cYghAAAAADGIIQAAAAAxiCEAAAAAMYghAAAAACAgICAgICAgICAgICA","0x6fe607b39661adc63400b3c0eeefa0fdde6031131c39a96800663250a75c667c":"5YMQAACgD9/MJLGyHXjvK3xlA+uTVGd3Q2hcLQChSotQKhd5EbA=","0x8cd6ed962850eebdb5bf360b496b5ab3425659a8ba3d115d5bb71055981a6bc2":"+F/GIIQtogABxiCEjigABMYghDwMaGXGIIQ1jGxsxiCEAQxoI8YghC2jAAHGIIQAQxAkxiCEPBC//8YghDYQ//DGIIQ0EQABxiCErgIACMYghK4RAATGIIQD4AAIgICAgA==","0xa0b2822e4ee4937ac10760ad5bb6108b3cfaef41c6d13342a1ba5fb3fd4c26ab":"+HGgt8fQkZe4faINYK98tz2Czvb1LZ/LaDPtw0E2aioQ4lagZcAk7Ttos/hqRPavUXnkrQVawEasLpVDVdR2xhGUexagjNbtlihQ7r21vzYLSWtas0JWWai6PRFdW7cQVZgaa8KAgICAgICAgICAgICAgA==","0xb7c7d09197b87da20d60af7cb73d82cef6f52d9fcb6833edc341366a2a10e256":"+HHGIIQ8EDAAxiCENhAQAMYghDwIRxfGIIQ1CDKFxiCErggAAMYghDwIqNfGIIQ1CDQexiCErggABMYghDwIXpfGIIQ1CC/GxiCErggACMYghDwIdyjGIIQ1CGOExiCErggADMYghDwI+ALGIIQ1CPjvgA=="}}; 7 | 8 | async function dynamicExecWithTrie(c, m, mm, root, preimages, rootdir) { 9 | let mdat = m.interface.encodeFunctionData("Step", [root]) 10 | let nodes 11 | 12 | while (true) { 13 | try { 14 | nodes = await getTrieNodesForCall(c, m.address, mdat, preimages) 15 | } catch(err) { 16 | if (err instanceof MissingHashError) { 17 | let value = fs.readFileSync(rootdir+"0x"+err.hash) 18 | console.log("handling hash oracle request") 19 | await mm.AddPreimage(value, err.offset) 20 | continue 21 | } else { 22 | throw err 23 | } 24 | } 25 | break 26 | } 27 | 28 | 29 | 30 | for (n of nodes) { 31 | await mm.AddTrieNode(n) 32 | } 33 | let ret = await m.Step(root) 34 | const receipt = await ret.wait() 35 | for (l of receipt.logs) { 36 | if (l.topics[0] == "0x86b89b5c9818dbbf520dd979a5f250d357508fe11b9511d4a43fd9bc6aa1be70") { 37 | root = l.data 38 | } 39 | } 40 | return root 41 | } 42 | 43 | // really a copy of mips_test_execwtrie 44 | describe("Exec with trie dynamic", function () { 45 | beforeEach(async function () { 46 | const MIPS = await ethers.getContractFactory("MIPS") 47 | m = await MIPS.deploy() 48 | mm = await ethers.getContractAt("MIPSMemory", await m.m()) 49 | 50 | // fake challenge for use of getTrieNodesForCall 51 | const Challenge = await ethers.getContractFactory("Challenge") 52 | const fakeGoldenHash = '0x0000000000000000000000000000000000000000000000000000000000000000' 53 | c = await Challenge.deploy(m.address, fakeGoldenHash) 54 | }) 55 | 56 | it("add should work", async function () { 57 | let root = trieAdd['root'] 58 | for (let i = 0; i < 12; i++) { 59 | root = await dynamicExecWithTrie(c, m, mm, root, trieAdd['preimages'], null) 60 | console.log(i, root) 61 | } 62 | }).timeout(120000) 63 | 64 | it("oracle should work", async function () { 65 | let root = trieOracle['root'] 66 | let pc = 0, out1, out2 67 | while (pc != 0x5ead0000) { 68 | root = await dynamicExecWithTrie(c, m, mm, root, trieOracle['preimages'], "mipsevm/testoracle/") 69 | 70 | pc = await mm.ReadMemory(root, 0xc0000080) 71 | out1 = await mm.ReadMemory(root, 0xbffffff4) 72 | out2 = await mm.ReadMemory(root, 0xbffffff8) 73 | console.log(root, pc, out1, out2) 74 | } 75 | expect(out1).to.equal(1) 76 | expect(out2).to.equal(1) 77 | }).timeout(300000) 78 | }) 79 | -------------------------------------------------------------------------------- /test/mips_test_memory.js: -------------------------------------------------------------------------------- 1 | const { expect } = require("chai"); 2 | const { writeMemory } = require("../scripts/lib") 3 | 4 | function randint(n) { 5 | return Math.floor(Math.random() * n) 6 | } 7 | 8 | describe("MIPSMemory contract", function () { 9 | beforeEach(async function () { 10 | const MIPSMemory = await ethers.getContractFactory("MIPSMemory") 11 | mm = await MIPSMemory.deploy() 12 | await mm.AddTrieNode(new Uint8Array([0x80])) 13 | }) 14 | it("write from new should work", async function() { 15 | let root = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" 16 | 17 | root = await writeMemory(mm, root, 0, 1) 18 | root = await writeMemory(mm, root, 4, 2) 19 | 20 | expect(await mm.ReadMemory(root, 0)).to.equal(1) 21 | expect(await mm.ReadMemory(root, 4)).to.equal(2) 22 | }) 23 | it("write three should work", async function() { 24 | await mm.AddTrieNode(new Uint8Array([0x80])) 25 | let root = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" 26 | 27 | root = await writeMemory(mm, root, 0, 1) 28 | root = await writeMemory(mm, root, 4, 2) 29 | root = await writeMemory(mm, root, 0x40, 3) 30 | 31 | expect(await mm.ReadMemory(root, 0)).to.equal(1) 32 | expect(await mm.ReadMemory(root, 4)).to.equal(2) 33 | expect(await mm.ReadMemory(root, 0x40)).to.equal(3) 34 | }) 35 | it("write other three should work", async function() { 36 | let root = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" 37 | 38 | root = await writeMemory(mm, root, 0x7fffd00c, 1) 39 | root = await writeMemory(mm, root, 0x7fffd010, 2) 40 | root = await writeMemory(mm, root, 0x7fffcffc, 3) 41 | 42 | expect(await mm.ReadMemory(root, 0x7fffd00c)).to.equal(1) 43 | expect(await mm.ReadMemory(root, 0x7fffd010)).to.equal(2) 44 | expect(await mm.ReadMemory(root, 0x7fffcffc)).to.equal(3) 45 | }) 46 | it("bug found fuzzing 1", async function() { 47 | let root = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" 48 | root = await writeMemory(mm, root, 0, 0) 49 | root = await writeMemory(mm, root, 0, 1) 50 | root = await writeMemory(mm, root, 0, 2) 51 | }) 52 | it("fuzzing should be okay", async function() { 53 | let root = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" 54 | let kv = {} 55 | 56 | for (var i = 0; i < 100; i++) { 57 | const keys = Object.keys(kv) 58 | const choice = Math.random() 59 | if (choice < 0.3 || keys.length == 0) { 60 | // write new key 61 | const key = randint(0x100)*4 62 | const value = randint(0x100000000) 63 | console.log("writing", key, value) 64 | root = await writeMemory(mm, root, key, value) 65 | kv[key] = value 66 | } else if (choice < 0.5) { 67 | // write new high key 68 | const key = randint(0x100)*4 + 0x10000000 69 | const value = randint(0x100000000) 70 | console.log("writing", key, value) 71 | root = await writeMemory(mm, root, key, value) 72 | kv[key] = value 73 | } else if (choice > 0.7) { 74 | // read old key 75 | const idx = randint(keys.length) 76 | const key = keys[idx] 77 | console.log("reading", key) 78 | expect(await mm.ReadMemory(root, key)).to.equal(kv[key]) 79 | } else { 80 | // rewrite old key 81 | const idx = randint(keys.length) 82 | const key = keys[idx] 83 | const value = randint(0x100000000) 84 | console.log("writing", key, value) 85 | root = await writeMemory(mm, root, key, value) 86 | kv[key] = value 87 | } 88 | } 89 | }).timeout(60000) 90 | }) 91 | -------------------------------------------------------------------------------- /test/mips_test_oracle.js: -------------------------------------------------------------------------------- 1 | const { keccak256 } = require("@ethersproject/keccak256"); 2 | const { expect } = require("chai"); 3 | 4 | const chai = require("chai"); 5 | const { solidity } = require("ethereum-waffle"); 6 | chai.use(solidity); 7 | 8 | const { writeMemory } = require("../scripts/lib") 9 | 10 | async function loadPreimageAndSelect(mm, data, offset) { 11 | // add in the preimage at offset 4 12 | const hash = keccak256(data) 13 | await mm.AddPreimage(data, offset) 14 | 15 | // write the oracle selection address 16 | let root = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" 17 | root = await writeMemory(mm, root, 0x30001000, hash, true) 18 | return root 19 | } 20 | 21 | describe("MIPSMemory oracle", function () { 22 | beforeEach(async function () { 23 | const MIPSMemory = await ethers.getContractFactory("MIPSMemory") 24 | mm = await MIPSMemory.deploy() 25 | await mm.AddTrieNode(new Uint8Array([0x80])) 26 | }) 27 | it("simple oracle", async function() { 28 | root = await loadPreimageAndSelect(mm, [0x11,0x22,0x33,0x44,0xaa,0xbb,0xcc,0xdd], 4) 29 | 30 | // length is 8 31 | expect(await mm.ReadMemory(root, 0x31000000)).to.equal(8) 32 | 33 | // offset 4 is 0xaabbccdd 34 | expect(await mm.ReadMemory(root, 0x31000008)).to.equal(0xaabbccdd) 35 | 36 | // offset 0 isn't loaded 37 | await expect(mm.ReadMemory(root, 0x31000004)).to.be.reverted; 38 | }) 39 | 40 | it("misaligned oracle", async function() { 41 | root = await loadPreimageAndSelect(mm, [0x11,0x22,0x33,0x44,0xaa,0xbb,0xcc], 4) 42 | expect(await mm.ReadMemory(root, 0x31000000)).to.equal(7) 43 | expect(await mm.ReadMemory(root, 0x31000008)).to.equal(0xaabbcc00) 44 | }) 45 | }) --------------------------------------------------------------------------------