├── README.md ├── bloom.py ├── casper.py ├── casper ├── README.md ├── casper.py ├── compute_scoring_rule_constants.py ├── distributions.py ├── networksim.py ├── run.py └── voting_strategy.py ├── casper3 ├── casper.py ├── distributions.py ├── networksim.py └── test.py ├── diffadjust ├── blkdiff.py └── hashpower.csv ├── erasure_code ├── share.cpp ├── share.go ├── share.h ├── share.js ├── share.py ├── share.pyc ├── test.py └── utils.h ├── ghost.py ├── ghost_timing_strats.py ├── inclusive_ghost.py ├── mining ├── arpow_miner.py ├── compute_probabilities_of_finality.py ├── finality_probability_sim.py ├── hashimoto.py ├── mining.go ├── mining.py └── python_sha3.py ├── multi_uncle_ghost.py ├── selfish_mining_strats.py ├── slasher_v2_sim.py ├── slasher_withholding_exploit.py ├── stability ├── csvgen.py ├── diff_and_price.csv ├── fit.py └── spread.py └── timing_strats.py /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethereum/economic-modeling/320caa16e0d253778131c6c933c7d7330a760634/README.md -------------------------------------------------------------------------------- /bloom.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | try: # 4 | shathree = __import__('sha3').sha3_256 5 | except: 6 | shathree = __import__('python_sha3').sha3_256 7 | 8 | 9 | params = { 10 | "size": 256, 11 | "pecks": 32 12 | } 13 | 14 | 15 | def decode_int(x): 16 | o = 0 17 | for a in x: 18 | o = o * 256 + ord(a) 19 | return o 20 | 21 | 22 | def sha3(x): 23 | return shathree(x).digest() 24 | 25 | 26 | def bloom_insert(params, bloom, val): 27 | k = decode_int(sha3(val)) * (3**160 + 112) 28 | for i in range(params["pecks"]): 29 | bloom |= 1 << (k % params["size"]) 30 | k //= params["size"] 31 | return bloom 32 | 33 | 34 | def bloom_query(params, bloom, val): 35 | o = bloom_insert(params, 0, val) 36 | return (bloom & o) == o 37 | 38 | 39 | def test_params(size, pecks, objcount): 40 | params = {"size": size, "pecks": pecks} 41 | count = 0 42 | for i in range(100): 43 | objs = [str(random.randrange(2**40)) for i in range(objcount)] 44 | bloom = 0 45 | for o in objs: 46 | bloom = bloom_insert(params, bloom, o) 47 | 48 | for o in objs: 49 | assert bloom_query(params, bloom, o) 50 | 51 | for i in range(100): 52 | if bloom_query(params, bloom, str(random.randrange(2**40))): 53 | count += 1 54 | 55 | print 'False positive rate: %f' % (count / 10000.) 56 | -------------------------------------------------------------------------------- /casper.py: -------------------------------------------------------------------------------- 1 | import copy, random, hashlib 2 | # GhostTable: { block number: { block: validators } } 3 | # The ghost table represents the entire current "view" of a user, and 4 | # every block produced contains the producer's ghost table at the time. 5 | 6 | # Signature slashing rules (not implemented) 7 | # 1. Sign two blocks at the same height 8 | # 2. Sign an invalid block 9 | # 3. Sign a block which fully confirms A at height H, and sign B at height H 10 | 11 | h = [3**50] 12 | ids = [0] 13 | 14 | # Number of validators 15 | NUM_VALIDATORS = 50 16 | # Block time in ticks (eg. 1 tick = 0.1 seconds) 17 | BLKTIME = 30 18 | # Disparity in the blocks of nodes 19 | CLOCK_DISPARITY = 10 20 | # An exponential distribution for latency offset by a minimum 21 | LATENCY_MIN = 4 22 | LATENCY_BASE = 2 23 | LATENCY_PROB = 0.25 24 | 25 | 26 | def assign_hash(): 27 | h[0] = (h[0] * 3) % 2**48 28 | return h[0] 29 | 30 | 31 | def assign_id(): 32 | ids[0] += 1 33 | return ids[0] - 1 34 | 35 | 36 | def latency_distribution_sample(): 37 | v = LATENCY_BASE 38 | while random.random() > LATENCY_PROB: 39 | v *= 2 40 | return v + LATENCY_MIN 41 | 42 | 43 | def clock_offset_distribution_sample(): 44 | return random.randrange(-CLOCK_DISPARITY, CLOCK_DISPARITY) 45 | 46 | 47 | # A signature represents the entire "view" of a signer, 48 | # where the view is the set of blocks that the signer 49 | # considered most likely to be valid at the time that 50 | # they were produced 51 | class Signature(): 52 | 53 | def __init__(self, signer, view): 54 | self.signer = signer 55 | self.view = copy.deepcopy(view) 56 | 57 | 58 | # A ghost table represents the view that a user had of the signatures 59 | # available at the time that the block was produced. 60 | class GhostTable(): 61 | 62 | def __init__(self): 63 | self.confirmed = [] 64 | self.unconfirmed = [] 65 | 66 | def process_signature(self, sig): 67 | # Process every block height in the signature 68 | for i in range(len(self.confirmed), len(sig.view)): 69 | # A ghost table entry at a height is a mapping of 70 | # block hash -> signers 71 | if i >= len(self.unconfirmed): 72 | self.unconfirmed.append({}) 73 | cur_entry = self.unconfirmed[i] 74 | # If the block hash is not yet in the ghost table, add it, and 75 | # initialize it with an empty signer set 76 | if sig.view[i] not in cur_entry: 77 | cur_entry[sig.view[i]] = {} 78 | # Add the signer 79 | cur_entry[sig.view[i]][sig.signer] = True 80 | # If it has 67% signatures, finalize 81 | if len(cur_entry[sig.view[i]]) > NUM_VALIDATORS * 2 / 3: 82 | # prevgt = block_map[sig.view[i]].gt 83 | prevgt = self 84 | print 'confirmed', block_map[sig.view[i]].height, sig.view[i] 85 | # Update blocks between the previous confirmation and the 86 | # current confirmation based on the newly confirmed block's 87 | # ghost table 88 | for j in range(len(self.confirmed), i): 89 | # At each intermediate height, add the block for which we 90 | # havethe most signatures 91 | maxkey, maxval = 0, 0 92 | for k in prevgt.unconfirmed[j]: 93 | if len(prevgt.unconfirmed[j][k]) > maxval: 94 | maxkey, maxval = k, len(prevgt.unconfirmed[j][k]) 95 | self.confirmed.append(maxkey) 96 | print j, {k: len(prevgt.unconfirmed[j][k]) for k in prevgt.unconfirmed[j]} 97 | # Then add the new block that got 67% signatures 98 | print i, sig.view[i] 99 | self.confirmed.append(sig.view[i]) 100 | 101 | # Hash of the ghost table's contents (to make sure that it's not 102 | # being modified when it's already supposed to be set in stone) 103 | def hash(self): 104 | print hashlib.sha256(repr(self.unconfirmed) + 105 | repr(self.confirmed)).hexdigest()[:15] 106 | 107 | # Create a new ghost table that appends to an existing ghost table, adding 108 | # some set of signatures 109 | def append(self, sigs): 110 | x = GhostTable() 111 | x.confirmed = copy.deepcopy(self.confirmed) 112 | x.unconfirmed = copy.deepcopy(self.unconfirmed) 113 | for sig in sigs: 114 | x.process_signature(sig) 115 | return x 116 | 117 | 118 | class Block(): 119 | 120 | def __init__(self, h, gt, maker): 121 | self.gt = gt 122 | self.height = h 123 | self.maker = maker 124 | self.hash = assign_hash() 125 | 126 | 127 | class Validator(): 128 | 129 | def __init__(self): 130 | self.gt = GhostTable() 131 | self.view = [] 132 | self.id = assign_id() 133 | self.new_sigs = [] 134 | self.clock_offset = clock_offset_distribution_sample() 135 | self.last_block_produced = -99999 136 | self.last_unseen = 0 137 | 138 | # Is this block compatible with our view? 139 | def is_compatible_with_view(self, block): 140 | return block.height >= len(self.view) or \ 141 | self.view[block.height] is None 142 | 143 | # Add a block to this validator's view of probably valid 144 | # blocks 145 | def add_to_view(self, block): 146 | while len(self.view) <= block.height: 147 | self.view.append(None) 148 | self.view[block.height] = block.hash 149 | while self.last_unseen < len(self.view) and \ 150 | self.view[self.last_unseen] is not None: 151 | self.last_unseen += 1 152 | 153 | # Make a block 154 | def produce_block(self): 155 | self.gt = self.gt.append(self.new_sigs) 156 | newblk = Block(self.last_unseen, self.gt, self.id) 157 | print 'newblk', newblk.height 158 | self.add_to_view(newblk) 159 | publish(newblk) 160 | newsig = Signature(self.id, self.view[:self.last_unseen]) 161 | self.new_sigs = [newsig] 162 | publish(newsig) 163 | 164 | # Callback function upon receiving a block 165 | def on_receive(self, obj): 166 | if isinstance(obj, Block): 167 | desired_maker = (self.time() // BLKTIME) % NUM_VALIDATORS 168 | if 0 <= (desired_maker - obj.maker) % 100 <= 0: 169 | if self.is_compatible_with_view(obj): 170 | self.add_to_view(obj) 171 | publish(Signature(self.id, self.view[:self.last_unseen])) 172 | if isinstance(obj, Signature): 173 | self.new_sigs.append(obj) 174 | 175 | # Do everything that you need to do in this particular round 176 | def tick(self): 177 | if (self.time() // BLKTIME) % NUM_VALIDATORS == self.id: 178 | if self.time() - self.last_block_produced > \ 179 | BLKTIME * NUM_VALIDATORS: 180 | self.produce_block() 181 | self.last_block_produced = self.time() 182 | 183 | # Calculate the validator's own clock based on the actual time 184 | # plus a time offset that this validator happens to be wrong by 185 | # (eg. +1 second) 186 | def time(self): 187 | return real_time[0] + self.clock_offset 188 | 189 | 190 | block_map = {} 191 | listening_queue = {} 192 | real_time = [0] 193 | validators = {} 194 | 195 | 196 | # Publish a block or a signature 197 | def publish(obj): 198 | if isinstance(obj, Block): 199 | block_map[obj.hash] = obj 200 | # For every validator, add it to the validator's listening queue 201 | # at a time randomly sampled from the latency distribution 202 | for v in validators: 203 | arrival_time = real_time[0] + latency_distribution_sample() 204 | if arrival_time not in listening_queue: 205 | listening_queue[arrival_time] = [] 206 | listening_queue[arrival_time].append((v, obj)) 207 | 208 | 209 | # One round of the clock ticking 210 | def tick(): 211 | for _, v in validators.items(): 212 | v.tick() 213 | if real_time[0] in listening_queue: 214 | for validator_id, obj in listening_queue[real_time[0]]: 215 | validators[validator_id].on_receive(obj) 216 | real_time[0] += 1 217 | print real_time[0] 218 | 219 | 220 | # Main function: run(7000) = simulate casper for 7000 ticks 221 | def run(steps): 222 | for k in block_map.keys(): 223 | del block_map[k] 224 | for k in listening_queue.keys(): 225 | del listening_queue[k] 226 | for k in validators.keys(): 227 | del validators[k] 228 | real_time[0] = 0 229 | ids[0] = 0 230 | for i in range(NUM_VALIDATORS): 231 | v = Validator() 232 | validators[v.id] = v 233 | for i in range(steps): 234 | tick() 235 | c = [] 236 | for _, v in validators.items(): 237 | for i, b in enumerate(v.gt.confirmed): 238 | assert block_map[b].height == i 239 | if v.gt.confirmed[:len(c)] != c[:len(v.gt.confirmed)]: 240 | for i in range(min(len(c), len(v.gt.confirmed))): 241 | if c[i] != v.gt.confirmed[i]: 242 | print i, c[i], v.gt.confirmed[i] 243 | raise Exception("Confirmed block list mismatch") 244 | c.extend(v.gt.confirmed[len(c):]) 245 | print c 246 | -------------------------------------------------------------------------------- /casper/README.md: -------------------------------------------------------------------------------- 1 | ### Casper proof of stake algorithm 2 | 3 | Casper is a new proof of stake algorithm being developed by Vitalik Buterin, Vlad Zamfir et al, which tries to resolve, or optimally compromise between, several of the outstanding issues in previous proof of stake algorithms. Proof of stake has many benefits, chief among which is the potential for massive reductions in electricity costs as well as more rapid convergence toward a state of "finality" where transactions cannot be reverted no matter what, but so far some of these issues have caused concern among developers and users, and led to a reduced willingness to adopt proof-of-stake protocols. 4 | 5 | The three primary issues with existing proof of stake algorithms can be described as follows. 6 | 7 | * **Nothing at stake**: if there are multiple "forks" of a blockchain, there is no incentive for validators not to register a vote on _every_ fork of the chain. This may end up a fatal problem even without any attackers, but if an attacker is present then they can, in theory, bribe everyone $0.001 to do this, and themselves mine only on their own fork, and thus revert an arbitrarily long chain at near-zero cost. 8 | 9 | Some blockchains (eg. [Tendermint](http://tendermint.com/)) solve the first problem via a "slashing" mechanic where validators are required to post security bonds, which can be taken away if a user double-votes. This insight can even be taken much further and provide a very strong robustness property where a double spend _cannot_ happen without a very large quantity of security bonds getting destroyed. However, to make this work, a requirement must be added where some portion `x > 0.5` of validators must either sign every block or sign the chain after some height, so that a successful double spend requires at least `2x - 1` of validators to sign two blocks (as `x` signed the original chain, `x` signed the new chain, and `2x - 1` signing both chains is the mathematically smallest possible overlap; you can see this yourself via Venn diagrams). Hence, `2x - 1` of all security deposits (eg. at `x = 2/3` this means one third) must be destroyed for a fork to happen. However, tis itself introduces two new problems: 10 | 11 | * **Sudden dropout ice age**: what if more than `1-x` of nodes suddenly drop dead? Then the blockchain may "freeze" forever. 12 | * **Stuck scenario**: what if, somehow, half of nodes sign one block and the other half sign the other block, so the chain cannot proceed further without a large quantity of deposits being destroyed? This is arguably as fatal a scenario as a successful double spend, and so getting to this point too should not be possible without a large quantity of security bond slashing. 13 | 14 | Casper introduces a proof of stake model which solves this using a similar (arguably cop-out) approach as Bitcoin: rather then trying to go from zero to finality in one step, we make it a gradual process, where validators progressively put more and more "stake" behind blocks until these blocks reach finality. The fundamental approach relies on a mechanism known as **consensus-by-bet**: essentially, instead of simply voting on which answer to give when the consensus mechanism requires a choice to be made, validators bet on the answer, and the bets themselves determine the answer. 15 | 16 | More precisely, at every block height, validators must achieve binary consensus (ie. all agree either 0 or 1) on the question: has a block been submitted at this height, and if so what block hash? We can model this by imagining a ball which starts off at the top of a hill, where there are multiple ramps down the hill that the ball can take: one ramp representing "no block" and one ramp representing "yes there is a block" (in the special malicious case where a validator submits two blocks, there may be two ramps representing "yes, there is a block and it's X" and "yes, there is a block and it's Y". The simple intuition behind the consensus algorithm is that each voter's strategy is: 17 | 18 | * Take the median position that everyone else is on (or, more precisely, mode by orientation and 33rd percentile by latitude) 19 | * Apply one step of "gravity" from this position. Randomly jiggle a bit to prevent "stuck" scenarios 20 | * Make this your new position, and publicly create a vote announcing this 21 | 22 | In the case that no block has been submitted, the "yes there is a block" ramp is closed off, and so there is only one ramp to go down. Once a block has fully gone down one particular ramp from the point of view of 2/3 of validators, consider it "finalized". This process takes place in parallel for every block height, and is continuous; validators can update their votes on every height every time a new block comes in. 23 | 24 | This algorithm can be analyzed from two perspectives: BFT analysis and economic analysis. From a BFT analysis perspective, the 33rd percentile rule makes the algorithm equivalent to a highly multi-round version of DLS-like pre-commit/commit/finalize algorithms. From an economic analysis perspective, the algorithm is much more interesting, and in fact borrows more from prediction market and scoring rule theory than it does from consensus theory. Every vote is a bet, and the bets are incentivized via a [scoring rule](https://en.wikipedia.org/wiki/Scoring_rule). The "altitude" on the ramps can be thought of as a probability estimate, where the top of the hill is 50% and the bottom of the hill is 99.99%; if you bet 97% on a particular outcome then that means that you are willing to accept a penalty of 97 if the result does NOT converge toward that outcome in exchange for a reward of 3 if the result does converge in that direction. The closer your bet gets to 100%, the more you gain if you are right, but the more you lose if you are wrong, until at 99.99% if you are wrong you lose your entire deposit. The idea is that validators get progressively more and more confident about a particular outcome as they see other validators getting more confident, and thus the system eventually (and in fact, logarithmically quickly) does converge on one outcome. 25 | 26 | The exact formula for this contains two functions, `s(p)` (which determines the payout given a bet `p` if you are right) and `f(p)` (which determines the payout given a bet `p` if you are wrong). To simplify things, we can also express the probability in terms of odds (which we'll denote `q`); for example, odds of 5:1 imply that something is five times more likely to happen than not happen, and in general we define `q = p/(1-p)`. In order for the formula to be incentive-compatible, we need (in probability form) `s'(p) * p = -f'(p) * (1-p)`, or (in odds form) `s'(q) * q = -f'(q)`. I propose the following functions: 27 | 28 | s(q) = q 29 | f(q) = -q^2 / 2 30 | 31 | As `q` gets high, you may notice that the penalties get high very quickly, though the rewards also increase albeit less drastically. For example, if your belief is that the odds of a given state (either 0 or 1) succeeding are 500:1, then if you bet `q = 400`, your expected return is `500/501 * s(400) + 1/501 * f(400) = 500/501 * 400 - 1/501 * 80000 = 239.52`, if you bet `q = 500` your expected return is 249.50, and if you bet `q = 600` your expected return is 239.52; if you bet `q = 1000` your expected return drops to zero and if you bet even more confidently your expected return becomes negative. In general, we expect risk aversion to lead to a bias where users converge toward an outcome more slowly than they "should" but this effect should only be moderately large; the "take the median of everyone else and apply one step of gravity" model is a good first approximation, and particularly ensures that you are not taking bets which are more than ~2.7x as risky/confident as bets that have already been taken by other validators. 32 | 33 | "Stuck" scenarios are impossible, as no one will vote confidently for a particular block unless a majority has already gone most of the way along the same direction; the worst outcome that may happen is one where the ball "remains at the top of the hill" for an extended period of time due to a combination of bad strategy selection and perhaps some malicious betting that tries to counteract any movements along any of the "ramps" going down. A possible counteraction is creating a default strategy that pushes harder and harder toward the "no block" outcome as time goes on. 34 | 35 | The sudden dropout ice age risk is dealt with via a compromise: if 67% of validators _are_ present, then the system converges; however, if fewer than 67% of validators are present then the system continues to produce blocks, but simply without agreeing on finality for any of them. Hence, in such a situation the system will simply have roughly the same level of security as systems like NXT. 36 | 37 | Ongoing research questions: 38 | 39 | * Do we detect such ice ages? If so, do we respond to them in a particular way (eg. reducing the 67% threshold over time)? Perhaps do we eventually switch to requiring only 67% of recently online nodes to agree on a block? Perhaps a proof of work backstop? 40 | * What is the most efficient way to express a signature? This algorithm carries a high data overhead, and so any reduction in the overhead would be highly appreciated. 41 | -------------------------------------------------------------------------------- /casper/compute_scoring_rule_constants.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | MAX_INTEREST_RATE = 4 * 10**-9 # 4 ppb/s, or 13.4% APR 4 | MIN_DEPOSIT_SIZE = 1500 5 | BLKTIME = 3 6 | WITHDRAWAL_PERIOD = 10**7 7 | TOTAL_DEPOSIT = 10**7 8 | 9 | MAX_BET_LENGTH = int(math.log(WITHDRAWAL_PERIOD / BLKTIME) / math.log(4)) 10 | MAX_RETURN = MAX_INTEREST_RATE * TOTAL_DEPOSIT * BLKTIME 11 | BET_MAXGAIN = MAX_RETURN / MAX_BET_LENGTH 12 | BET_MAXLOSS = MIN_DEPOSIT_SIZE 13 | 14 | # Best way I could find to compute the fixed point 15 | MAXODDS = BET_MAXLOSS / (BET_MAXGAIN / 4) 16 | MAXODDS = BET_MAXLOSS / (BET_MAXGAIN/4 + BET_MAXGAIN / (2*math.log(MAXODDS))) 17 | MAXODDS = BET_MAXLOSS / (BET_MAXGAIN/4 + BET_MAXGAIN / (2*math.log(MAXODDS))) 18 | 19 | B = BET_MAXGAIN / (2 * MAXODDS) 20 | A = MAXODDS / math.log(MAXODDS) * B 21 | # The above equations determine parameters fitted such that all three 22 | # of the following are true: 23 | # 24 | # 1. 50% of the winnings come from a logarithmic component, 50% from a linear 25 | # component 26 | # 2. The max winnings are set as above 27 | # 3. The max losses are set as above 28 | 29 | print A, B, MAXODDS, MAX_BET_LENGTH 30 | 31 | def score_correct(logodds): 32 | odds = MAXODDS**(logodds / 255.) 33 | return math.log(odds) * A + odds * B 34 | 35 | def score_incorrect(logodds): 36 | odds = MAXODDS**(logodds / 255.) 37 | return -odds * A - odds**2/2 * B 38 | 39 | print score_correct(255) 40 | print score_incorrect(255) 41 | -------------------------------------------------------------------------------- /casper/distributions.py: -------------------------------------------------------------------------------- 1 | import random, sys 2 | 3 | 4 | def normal_distribution(mean, standev): 5 | def f(): 6 | return int(random.normalvariate(mean, standev)) 7 | 8 | return f 9 | 10 | 11 | def exponential_distribution(mean): 12 | def f(): 13 | total = 0 14 | while 1: 15 | total += 1 16 | if not random.randrange(32): 17 | break 18 | return int(total * 0.03125 * mean) 19 | 20 | return f 21 | 22 | 23 | def convolve(*args): 24 | def f(): 25 | total = 0 26 | for arg in args: 27 | total += arg() 28 | return total 29 | 30 | return f 31 | 32 | 33 | def transform(dist, xformer): 34 | def f(): 35 | return xformer(dist()) 36 | 37 | return f 38 | -------------------------------------------------------------------------------- /casper/networksim.py: -------------------------------------------------------------------------------- 1 | from distributions import transform, normal_distribution 2 | import random 3 | 4 | 5 | class NetworkSimulator(): 6 | 7 | def __init__(self, latency=50): 8 | self.agents = [] 9 | self.latency_distribution_sample = transform(normal_distribution(latency, (latency * 2) // 5), lambda x: max(x, 0)) 10 | self.time = 0 11 | self.objqueue = {} 12 | self.peers = {} 13 | self.reliability = 0.9 14 | 15 | def generate_peers(self, num_peers=5): 16 | self.peers = {} 17 | for a in self.agents: 18 | p = [] 19 | while len(p) <= num_peers // 2: 20 | p.append(random.choice(self.agents)) 21 | if p[-1] == a: 22 | p.pop() 23 | self.peers[a.id] = self.peers.get(a.id, []) + p 24 | for peer in p: 25 | self.peers[peer.id] = self.peers.get(peer.id, []) + [a] 26 | 27 | def tick(self): 28 | if self.time in self.objqueue: 29 | for recipient, obj in self.objqueue[self.time]: 30 | if random.random() < self.reliability: 31 | recipient.on_receive(obj) 32 | del self.objqueue[self.time] 33 | for a in self.agents: 34 | a.tick() 35 | self.time += 1 36 | 37 | def run(self, steps): 38 | for i in range(steps): 39 | self.tick() 40 | 41 | def broadcast(self, sender, obj): 42 | for p in self.peers[sender.id]: 43 | recv_time = self.time + self.latency_distribution_sample() 44 | if recv_time not in self.objqueue: 45 | self.objqueue[recv_time] = [] 46 | self.objqueue[recv_time].append((p, obj)) 47 | 48 | def direct_send(self, to_id, obj): 49 | for a in self.agents: 50 | if a.id == to_id: 51 | recv_time = self.time + self.latency_distribution_sample() 52 | if recv_time not in self.objqueue: 53 | self.objqueue[recv_time] = [] 54 | self.objqueue[recv_time].append((a, obj)) 55 | 56 | def knock_offline_random(self, n): 57 | ko = {} 58 | while len(ko) < n: 59 | c = random.choice(self.agents) 60 | ko[c.id] = c 61 | for c in ko.values(): 62 | self.peers[c.id] = [] 63 | for a in self.agents: 64 | self.peers[a.id] = [x for x in self.peers[a.id] if x.id not in ko] 65 | 66 | def partition(self): 67 | a = {} 68 | while len(a) < len(self.agents) / 2: 69 | c = random.choice(self.agents) 70 | a[c.id] = c 71 | for c in self.agents: 72 | if c.id in a: 73 | self.peers[c.id] = [x for x in self.peers[c.id] if x.id in a] 74 | else: 75 | self.peers[c.id] = [x for x in self.peers[c.id] if x.id not in a] 76 | -------------------------------------------------------------------------------- /casper/run.py: -------------------------------------------------------------------------------- 1 | import casper 2 | import sys 3 | 4 | casper.logging_level = int(sys.argv[1]) if len(sys.argv) > 1 else 0 5 | casper.run(50000) 6 | -------------------------------------------------------------------------------- /casper/voting_strategy.py: -------------------------------------------------------------------------------- 1 | import random 2 | import math 3 | 4 | # The voting strategy. Validators see what every other validator votes, 5 | # and return their vote. 6 | # 7 | # Remember, 0 and 1 are not probabilities! 8 | # http://lesswrong.com/lw/mp/0_and_1_are_not_probabilities/ !) 9 | 10 | def default_vote(scheduled_time, received_time, now, **kwargs): 11 | if received_time is None: 12 | time_delta = now - scheduled_time 13 | my_opinion_prob = 1 if time_delta < kwargs["blktime"] * 4 else 4.0 / (4 + time_delta * 1.0 / kwargs["blktime"]) 14 | return 0.5 if random.random() < my_opinion_prob else 0.3 15 | else: 16 | time_delta = received_time * 0.98 + now * 0.02 - scheduled_time 17 | my_opinion_prob = 1 if abs(time_delta) < kwargs["blktime"] * 4 else 4.0 / (4 + abs(time_delta) * 1.0 / kwargs["blktime"]) 18 | return 0.7 if random.random() < my_opinion_prob else 0.3 19 | 20 | 21 | def vote(probs): 22 | if len(probs) == 0: 23 | return 0.5 24 | probs = sorted(probs) 25 | if probs[len(probs)/3] >= 0.7: 26 | return 0.84 + probs[len(probs)/3] * 0.16 27 | elif probs[len(probs)*2/3] <= 0.3: 28 | return probs[len(probs)*2/3] * 0.16 29 | else: 30 | return max(0.2, min(0.8, probs[len(probs)/2] * 3 - 0.8 - random.random() * 0.4)) 31 | 32 | FINALITY_THRESHOLD = 0.000001 33 | 34 | def aggressive_vote(probs): 35 | if len(probs) == 0: 36 | return 0.5 37 | probs = sorted(probs) 38 | if probs[len(probs)/3] >= 0.9: 39 | return 1 - FINALITY_THRESHOLD 40 | elif probs[len(probs)*2/3] <= 0.1: 41 | return FINALITY_THRESHOLD 42 | if probs[len(probs)/3] >= 0.7: 43 | return 0.955 44 | elif probs[len(probs)*2/3] <= 0.3: 45 | return 0.045 46 | else: 47 | return max(0.2, min(0.8, probs[len(probs)/2] * 3 - 0.8 - random.random() * 0.4)) 48 | 49 | def craycray_vote(probs): 50 | if len(probs) == 0: 51 | return 0.5 52 | probs = sorted(probs) 53 | if probs[len(probs)/3] >= 1 - FINALITY_THRESHOLD: 54 | return 1 - FINALITY_THRESHOLD 55 | elif probs[len(probs)*2/3] <= FINALITY_THRESHOLD: 56 | return FINALITY_THRESHOLD 57 | elif random.choice((0, 1)): 58 | o = FINALITY_THRESHOLD * 2 + (1 - FINALITY_THRESHOLD * 4) * 0.5 ** random.randrange(0, int(math.log(1 / FINALITY_THRESHOLD) / math.log(2))) 59 | return o 60 | else: 61 | o = 1 - FINALITY_THRESHOLD * 2 - (1 - FINALITY_THRESHOLD * 4) * 0.5 ** random.randrange(0, int(math.log(1 / FINALITY_THRESHOLD) / math.log(2))) 62 | return o 63 | -------------------------------------------------------------------------------- /casper3/casper.py: -------------------------------------------------------------------------------- 1 | from ethereum.casper_utils import RandaoManager, get_skips_and_block_making_time, \ 2 | generate_validation_code, call_casper, make_block, check_skips, get_timestamp, \ 3 | get_casper_ct, validator_sizes 4 | from ethereum.utils import sha3, hash32, privtoaddr, ecsign, zpad, encode_int32, \ 5 | big_endian_to_int 6 | from ethereum.block import Block 7 | from ethereum.transactions import Transaction 8 | from ethereum.chain import Chain 9 | import networksim 10 | import rlp 11 | import random 12 | 13 | CHECK_FOR_UNCLES_BACK = 8 14 | 15 | global_block_counter = 0 16 | 17 | casper_ct = get_casper_ct() 18 | 19 | class ChildRequest(rlp.Serializable): 20 | fields = [ 21 | ('prevhash', hash32) 22 | ] 23 | 24 | def __init__(self, prevhash): 25 | self.prevhash = prevhash 26 | 27 | @property 28 | def hash(self): 29 | return sha3(self.prevhash + '::salt:jhfqou213nry138o2r124124') 30 | 31 | ids = [] 32 | 33 | class Validator(): 34 | def __init__(self, genesis, key, network, env, time_offset=5): 35 | # Create a chain object 36 | self.chain = Chain(genesis, env=env) 37 | # Use the validator's time as the chain's time 38 | self.chain.time = lambda: self.get_timestamp() 39 | # My private key 40 | self.key = key 41 | # My address 42 | self.address = privtoaddr(key) 43 | # My randao 44 | self.randao = RandaoManager(sha3(self.key)) 45 | # Pointer to the test p2p network 46 | self.network = network 47 | # Record of objects already received and processed 48 | self.received_objects = {} 49 | # The minimum eligible timestamp given a particular number of skips 50 | self.next_skip_count = 0 51 | self.next_skip_timestamp = 0 52 | # This validator's indices in the state 53 | self.indices = None 54 | # Is this validator active? 55 | self.active = False 56 | # Code that verifies signatures from this validator 57 | self.validation_code = generate_validation_code(privtoaddr(key)) 58 | # Parents that this validator has already built a block on 59 | self.used_parents = {} 60 | # This validator's clock offset (for testing purposes) 61 | self.time_offset = random.randrange(time_offset) - (time_offset // 2) 62 | # Determine the epoch length 63 | self.epoch_length = self.call_casper('getEpochLength') 64 | # Give this validator a unique ID 65 | self.id = len(ids) 66 | ids.append(self.id) 67 | self.find_my_indices() 68 | self.cached_head = self.chain.head_hash 69 | 70 | def call_casper(self, fun, args=[]): 71 | return call_casper(self.chain.state, fun, args) 72 | 73 | def find_my_indices(self): 74 | epoch = self.chain.state.block_number // self.epoch_length 75 | print 'Finding indices for epoch %d' % epoch, self.call_casper('getEpoch') 76 | for i in range(len(validator_sizes)): 77 | valcount = self.call_casper('getHistoricalValidatorCount', [epoch, i]) 78 | print i, valcount, self.call_casper('getHistoricalValidatorCount', [0, i]) 79 | for j in range(valcount): 80 | valcode = self.call_casper('getValidationCode', [i, j]) 81 | print (valcode, self.validation_code) 82 | if valcode == self.validation_code: 83 | self.indices = i, j 84 | start = self.call_casper('getStartEpoch', [i, j]) 85 | end = self.call_casper('getEndEpoch', [i, j]) 86 | if start <= epoch < end: 87 | self.active = True 88 | self.next_skip_count = 0 89 | self.next_skip_timestamp = get_timestamp(self.chain, self.next_skip_count) 90 | print 'In current validator set at (%d, %d)' % (i, j) 91 | return 92 | else: 93 | self.indices = None 94 | self.active = False 95 | self.next_skip_count, self.next_skip_timestamp = 0, 0 96 | print 'Registered at (%d, %d) but not in current set' % (i, j) 97 | return 98 | self.indices = None 99 | self.active = False 100 | self.next_skip_count, self.next_skip_timestamp = 0, 0 101 | print 'Not in current validator set' 102 | 103 | def get_uncles(self): 104 | anc = self.chain.get_block(self.chain.get_blockhash_by_number(self.chain.state.block_number - CHECK_FOR_UNCLES_BACK)) 105 | if anc: 106 | descendants = self.chain.get_descendants(anc) 107 | else: 108 | descendants = self.chain.get_descendants(self.chain.db.get('GENESIS_HASH')) 109 | potential_uncles = [x for x in descendants if x not in self.chain and isinstance(x, Block)] 110 | uncles = [x.header for x in potential_uncles if not call_casper(self.chain.state, 'isDunkleIncluded', [x.header.hash])] 111 | return uncles 112 | 113 | def get_timestamp(self): 114 | return int(self.network.time * 0.01) + self.time_offset 115 | 116 | def on_receive(self, obj): 117 | if isinstance(obj, list): 118 | for _obj in obj: 119 | self.on_receive(_obj) 120 | return 121 | if obj.hash in self.received_objects: 122 | return 123 | if isinstance(obj, Block): 124 | print 'Receiving block', obj 125 | assert obj.hash not in self.chain 126 | block_success = self.chain.add_block(obj) 127 | self.network.broadcast(self, obj) 128 | self.network.broadcast(self, ChildRequest(obj.header.hash)) 129 | self.update_head() 130 | elif isinstance(obj, Transaction): 131 | if self.chain.add_transaction(obj): 132 | self.network.broadcast(self, obj) 133 | self.received_objects[obj.hash] = True 134 | for x in self.chain.get_chain(): 135 | assert x.hash in self.received_objects 136 | 137 | def tick(self): 138 | # Try to create a block 139 | # Conditions: 140 | # (i) you are an active validator, 141 | # (ii) you have not yet made a block with this parent 142 | if self.indices and self.chain.head_hash not in self.used_parents: 143 | t = self.get_timestamp() 144 | # Is it early enough to create the block? 145 | if t >= self.next_skip_timestamp and (not self.chain.head or t > self.chain.head.header.timestamp): 146 | # Wrong validator; in this case, just wait for the next skip count 147 | if not check_skips(self.chain, self.indices, self.next_skip_count): 148 | self.next_skip_count += 1 149 | self.next_skip_timestamp = get_timestamp(self.chain, self.next_skip_count) 150 | # print 'Incrementing proposed timestamp for block %d to %d' % \ 151 | # (self.chain.head.header.number + 1 if self.chain.head else 0, self.next_skip_timestamp) 152 | return 153 | self.used_parents[self.chain.head_hash] = True 154 | # Simulated 15% chance of validator failure to make a block 155 | if random.random() > 0.999: 156 | print 'Simulating validator failure, block %d not created' % (self.chain.head.header.number + 1 if self.chain.head else 0) 157 | return 158 | # Make the block, make sure it's valid 159 | pre_dunkle_count = call_casper(self.chain.state, 'getTotalDunklesIncluded', []) 160 | dunkle_txs = [] 161 | for i, u in enumerate(self.get_uncles()[:4]): 162 | start_nonce = self.chain.state.get_nonce(self.address) 163 | txdata = casper_ct.encode('includeDunkle', [rlp.encode(u)]) 164 | dunkle_txs.append(Transaction(start_nonce + i, 0, 650000, self.chain.config['CASPER_ADDR'], 0, txdata).sign(self.key)) 165 | for dtx in dunkle_txs[::-1]: 166 | self.chain.add_transaction(dtx, force=True) 167 | blk = make_block(self.chain, self.key, self.randao, self.indices, self.next_skip_count) 168 | global global_block_counter 169 | global_block_counter += 1 170 | for dtx in dunkle_txs: 171 | assert dtx in blk.transactions, (dtx, blk.transactions) 172 | print 'made block with timestamp %d and %d dunkles' % (blk.timestamp, len(dunkle_txs)) 173 | assert blk.timestamp >= self.next_skip_timestamp 174 | assert self.chain.add_block(blk) 175 | self.update_head() 176 | post_dunkle_count = call_casper(self.chain.state, 'getTotalDunklesIncluded', []) 177 | assert post_dunkle_count - pre_dunkle_count == len(dunkle_txs) 178 | self.received_objects[blk.hash] = True 179 | print 'Validator %d making block %d (%s)' % (self.id, blk.header.number, blk.header.hash[:8].encode('hex')) 180 | self.network.broadcast(self, blk) 181 | # Sometimes we received blocks too early or out of order; 182 | # run an occasional loop that processes these 183 | if random.random() < 0.02: 184 | self.chain.process_time_queue() 185 | self.chain.process_parent_queue() 186 | self.update_head() 187 | 188 | def update_head(self): 189 | if self.cached_head == self.chain.head_hash: 190 | return 191 | self.cached_head = self.chain.head_hash 192 | if self.chain.state.block_number % self.epoch_length == 0: 193 | self.find_my_indices() 194 | if self.indices: 195 | self.next_skip_count = 0 196 | self.next_skip_timestamp = get_timestamp(self.chain, self.next_skip_count) 197 | print 'Head changed: %s, will attempt creating a block at %d' % (self.chain.head_hash.encode('hex'), self.next_skip_timestamp) 198 | 199 | def withdraw(self, gasprice=20 * 10**9): 200 | h = sha3(b'withdrawwithdrawwithdrawwithdraw') 201 | v, r, s = ecsign(h, self.key) 202 | sigdata = encode_int32(v) + encode_int32(r) + encode_int32(s) 203 | txdata = casper_ct.encode('startWithdrawal', [self.indices[0], self.indices[1], sigdata]) 204 | tx = Transaction(self.chain.state.get_nonce(self.address), gasprice, 650000, self.chain.config['CASPER_ADDR'], 0, txdata).sign(self.key) 205 | self.chain.add_transaction(tx) 206 | self.network.broadcast(self, tx) 207 | print 'Withdrawing!' 208 | -------------------------------------------------------------------------------- /casper3/distributions.py: -------------------------------------------------------------------------------- 1 | import random, sys 2 | 3 | 4 | def normal_distribution(mean, standev): 5 | def f(): 6 | return int(random.normalvariate(mean, standev)) 7 | 8 | return f 9 | 10 | 11 | def exponential_distribution(mean): 12 | def f(): 13 | total = 0 14 | while 1: 15 | total += 1 16 | if not random.randrange(32): 17 | break 18 | return int(total * 0.03125 * mean) 19 | 20 | return f 21 | 22 | 23 | def convolve(*args): 24 | def f(): 25 | total = 0 26 | for arg in args: 27 | total += arg() 28 | return total 29 | 30 | return f 31 | 32 | 33 | def transform(dist, xformer): 34 | def f(): 35 | return xformer(dist()) 36 | 37 | return f 38 | -------------------------------------------------------------------------------- /casper3/networksim.py: -------------------------------------------------------------------------------- 1 | from distributions import transform, normal_distribution 2 | import random 3 | 4 | 5 | class NetworkSimulator(): 6 | 7 | def __init__(self, latency=50): 8 | self.agents = [] 9 | self.latency_distribution_sample = transform(normal_distribution(latency, (latency * 2) // 5), lambda x: max(x, 0)) 10 | self.time = 0 11 | self.objqueue = {} 12 | self.peers = {} 13 | self.reliability = 0.9 14 | 15 | def generate_peers(self, num_peers=5): 16 | self.peers = {} 17 | for a in self.agents: 18 | p = [] 19 | while len(p) <= num_peers // 2: 20 | p.append(random.choice(self.agents)) 21 | if p[-1] == a: 22 | p.pop() 23 | self.peers[a.id] = self.peers.get(a.id, []) + p 24 | for peer in p: 25 | self.peers[peer.id] = self.peers.get(peer.id, []) + [a] 26 | 27 | def tick(self): 28 | if self.time in self.objqueue: 29 | for recipient, obj in self.objqueue[self.time]: 30 | if random.random() < self.reliability: 31 | recipient.on_receive(obj) 32 | del self.objqueue[self.time] 33 | for a in self.agents: 34 | a.tick() 35 | self.time += 1 36 | 37 | def run(self, steps): 38 | for i in range(steps): 39 | self.tick() 40 | 41 | def broadcast(self, sender, obj): 42 | for p in self.peers[sender.id]: 43 | recv_time = self.time + self.latency_distribution_sample() 44 | if recv_time not in self.objqueue: 45 | self.objqueue[recv_time] = [] 46 | self.objqueue[recv_time].append((p, obj)) 47 | 48 | def direct_send(self, to_id, obj): 49 | for a in self.agents: 50 | if a.id == to_id: 51 | recv_time = self.time + self.latency_distribution_sample() 52 | if recv_time not in self.objqueue: 53 | self.objqueue[recv_time] = [] 54 | self.objqueue[recv_time].append((a, obj)) 55 | 56 | def knock_offline_random(self, n): 57 | ko = {} 58 | while len(ko) < n: 59 | c = random.choice(self.agents) 60 | ko[c.id] = c 61 | for c in ko.values(): 62 | self.peers[c.id] = [] 63 | for a in self.agents: 64 | self.peers[a.id] = [x for x in self.peers[a.id] if x.id not in ko] 65 | 66 | def partition(self): 67 | a = {} 68 | while len(a) < len(self.agents) / 2: 69 | c = random.choice(self.agents) 70 | a[c.id] = c 71 | for c in self.agents: 72 | if c.id in a: 73 | self.peers[c.id] = [x for x in self.peers[c.id] if x.id in a] 74 | else: 75 | self.peers[c.id] = [x for x in self.peers[c.id] if x.id not in a] 76 | -------------------------------------------------------------------------------- /casper3/test.py: -------------------------------------------------------------------------------- 1 | import networksim 2 | from casper import Validator 3 | import casper 4 | from ethereum.parse_genesis_declaration import mk_basic_state 5 | from ethereum.config import Env 6 | from ethereum.casper_utils import RandaoManager, generate_validation_code, call_casper, \ 7 | get_skips_and_block_making_time, sign_block, make_block, get_contract_code, \ 8 | casper_config, get_casper_ct, get_casper_code, get_rlp_decoder_code, \ 9 | get_hash_without_ed_code, make_casper_genesis, validator_sizes, find_indices 10 | from ethereum.utils import sha3, privtoaddr 11 | from ethereum.transactions import Transaction 12 | from ethereum.state_transition import apply_transaction 13 | 14 | from ethereum.slogging import LogRecorder, configure_logging, set_level 15 | # config_string = ':info,eth.vm.log:trace,eth.vm.op:trace,eth.vm.stack:trace,eth.vm.exit:trace,eth.pb.msg:trace,eth.pb.tx:debug' 16 | config_string = ':info,eth.vm.log:trace' 17 | configure_logging(config_string=config_string) 18 | 19 | n = networksim.NetworkSimulator(latency=150) 20 | n.time = 2 21 | print 'Generating keys' 22 | keys = [sha3(str(i)) for i in range(20)] 23 | print 'Initializing randaos' 24 | randaos = [RandaoManager(sha3(k)) for k in keys] 25 | deposit_sizes = [128] * 15 + [256] * 5 26 | 27 | print 'Creating genesis state' 28 | s = make_casper_genesis(validators=[(generate_validation_code(privtoaddr(k)), ds * 10**18, r.get(9999)) 29 | for k, ds, r in zip(keys, deposit_sizes, randaos)], 30 | alloc={privtoaddr(k): {'balance': 10**18} for k in keys}, 31 | timestamp=2, 32 | epoch_length=40) 33 | g = s.to_snapshot() 34 | print 'Genesis state created' 35 | 36 | validators = [Validator(g, k, n, Env(config=casper_config), time_offset=4) for k in keys] 37 | n.agents = validators 38 | n.generate_peers() 39 | lowest_shared_height = -1 40 | made_101_check = 0 41 | 42 | for i in range(100000): 43 | # print 'ticking' 44 | n.tick() 45 | if i % 100 == 0: 46 | print '%d ticks passed' % i 47 | print 'Validator heads:', [v.chain.head.header.number if v.chain.head else None for v in validators] 48 | print 'Total blocks created:', casper.global_block_counter 49 | print 'Dunkle count:', call_casper(validators[0].chain.state, 'getTotalDunklesIncluded', []) 50 | lowest_shared_height = min([v.chain.head.header.number if v.chain.head else -1 for v in validators]) 51 | if lowest_shared_height >= 101 and not made_101_check: 52 | made_101_check = True 53 | print 'Checking that withdrawn validators are inactive' 54 | assert len([v for v in validators if v.active]) == len(validators) - 5, len([v for v in validators if v.active]) 55 | print 'Check successful' 56 | break 57 | if i == 1: 58 | print 'Checking that all validators are active' 59 | assert len([v for v in validators if v.active]) == len(validators) 60 | print 'Check successful' 61 | if i == 2000: 62 | print 'Withdrawing a few validators' 63 | for v in validators[:5]: 64 | v.withdraw() 65 | if i == 4000: 66 | print 'Checking that validators have withdrawn' 67 | for v in validators[:5]: 68 | assert call_casper(v.chain.state, 'getEndEpoch', []) <= 2 69 | print 'Check successful' 70 | -------------------------------------------------------------------------------- /diffadjust/blkdiff.py: -------------------------------------------------------------------------------- 1 | import math, random 2 | 3 | hashpower = [float(x) for x in open('hashpower.csv').readlines()] 4 | 5 | # Target block time 6 | TARGET = 12 7 | # Should be 86400, but can reduce for a quicker sim 8 | SECONDS_IN_DAY = 86400 9 | # Look at the 1/x day exponential moving average 10 | EMA_FACTOR = 0.01 11 | # Damping factor for simple difficulty adjustment 12 | SIMPLE_ADJUST_DAMPING_FACTOR = 20 13 | # Maximum per-block diff adjustment (as fraction of current diff) 14 | SIMPLE_ADJUST_MAX = 0.5 15 | # Damping factor for quadratic difficulty adjustment 16 | QUADRATIC_ADJUST_DAMPING_FACTOR = 3 17 | # Maximum per-block diff adjustment (as fraction of current diff) 18 | QUADRATIC_ADJUST_MAX = 0.5 19 | # Threshold for bounded adjustor 20 | BOUNDED_ADJUST_THRESHOLD = 1.3 21 | # Bounded adjustment factor 22 | BOUNDED_ADJUST_FACTOR = 0.01 23 | # How many blocks back to look 24 | BLKS_BACK = 10 25 | # Naive difficulty adjustment factor 26 | NAIVE_ADJUST_FACTOR = 1/1024. 27 | 28 | 29 | # Produces a value according to the exponential distribution; used 30 | # to determine the time until the next block given an average block 31 | # time of t 32 | def expdiff(t): 33 | return -math.log(random.random()) * t 34 | 35 | 36 | # abs_sqr(3) = 9, abs_sqr(-7) = -49, etc 37 | def abs_sqr(x): 38 | return -(x**2) if x < 0 else x**2 39 | 40 | 41 | # Given an array of the most recent timestamps, and the most recent 42 | # difficulties, compute the next difficulty 43 | def simple_adjust(timestamps, diffs): 44 | if len(timestamps) < BLKS_BACK + 2: 45 | return diffs[-1] 46 | # Total interval between previous block and block a bit further back 47 | delta = timestamps[-2] - timestamps[-2-BLKS_BACK] + 0.0 48 | # Expected interval 49 | expected = TARGET * BLKS_BACK 50 | # Compute adjustment factor 51 | fac = 1 - (delta / expected - 1) / SIMPLE_ADJUST_DAMPING_FACTOR 52 | fac = max(min(fac, 1 + SIMPLE_ADJUST_MAX), 1 - SIMPLE_ADJUST_MAX) 53 | return diffs[-1] * fac 54 | 55 | 56 | # Alternative adjustment algorithm 57 | def quadratic_adjust(timestamps, diffs): 58 | if len(timestamps) < BLKS_BACK + 2: 59 | return diffs[-1] 60 | # Total interval between previous block and block a bit further back 61 | delta = timestamps[-2] - timestamps[-2-BLKS_BACK] + 0.0 62 | # Expected interval 63 | expected = TARGET * BLKS_BACK 64 | # Compute adjustment factor 65 | fac = 1 - abs_sqr(delta / expected - 1) / QUADRATIC_ADJUST_DAMPING_FACTOR 66 | fac = max(min(fac, 1 + QUADRATIC_ADJUST_MAX), 1 - QUADRATIC_ADJUST_MAX) 67 | return diffs[-1] * fac 68 | 69 | 70 | # Alternative adjustment algorithm 71 | def bounded_adjust(timestamps, diffs): 72 | if len(timestamps) < BLKS_BACK + 2: 73 | return diffs[-1] 74 | # Total interval between previous block and block a bit further back 75 | delta = timestamps[-2] - timestamps[-2-BLKS_BACK] + 0.0 76 | # Expected interval 77 | expected = TARGET * BLKS_BACK 78 | if delta / expected > BOUNDED_ADJUST_THRESHOLD: 79 | fac = (1 - BOUNDED_ADJUST_FACTOR) 80 | elif delta / expected < 1 / BOUNDED_ADJUST_THRESHOLD: 81 | fac = (1 + BOUNDED_ADJUST_FACTOR) ** (delta / expected) 82 | else: 83 | fac = 1 84 | return diffs[-1] * fac 85 | 86 | 87 | # Old Ethereum algorithm 88 | def old_adjust(timestamps, diffs): 89 | if len(timestamps) < 2: 90 | return diffs[-1] 91 | delta = timestamps[-1] - timestamps[-2] 92 | expected = TARGET * 0.693 93 | if delta > expected: 94 | fac = 1 - NAIVE_ADJUST_FACTOR 95 | else: 96 | fac = 1 + NAIVE_ADJUST_FACTOR 97 | return diffs[-1] * fac 98 | 99 | 100 | def test(source, adjust): 101 | # Variables to keep track of for stats purposes 102 | ema = maxema = minema = TARGET 103 | lthalf, gtdouble, lttq, gtft = 0, 0, 0, 0 104 | count = 0 105 | # Block times 106 | times = [0] 107 | # Block difficulty values 108 | diffs = [source[0]] 109 | # Next time to print status update 110 | nextprint = 10**6 111 | # Main loop 112 | while times[-1] < len(source) * SECONDS_IN_DAY: 113 | # Print status update every 10**6 seconds 114 | if times[-1] > nextprint: 115 | print '%d out of %d processed, ema %f' % \ 116 | (times[-1], len(source) * SECONDS_IN_DAY, ema) 117 | nextprint += 10**6 118 | # Grab hashpower from data source 119 | hashpower = source[int(times[-1] // SECONDS_IN_DAY)] 120 | # Calculate new difficulty 121 | diffs.append(adjust(times, diffs)) 122 | # Calculate next block time 123 | times.append(times[-1] + expdiff(diffs[-1] / hashpower)) 124 | # Calculate min and max ema 125 | ema = ema * (1 - EMA_FACTOR) + (times[-1] - times[-2]) * EMA_FACTOR 126 | minema = min(minema, ema) 127 | maxema = max(maxema, ema) 128 | count += 1 129 | # Keep track of number of blocks we are below 75/50% or above 130 | # 133/200% of target 131 | if ema < TARGET * 0.75: 132 | lttq += 1 133 | if ema < TARGET * 0.5: 134 | lthalf += 1 135 | elif ema > TARGET * 1.33333: 136 | gtft += 1 137 | if ema > TARGET * 2: 138 | gtdouble += 1 139 | # Pop items to save memory 140 | if len(times) > 2000: 141 | times.pop(0) 142 | diffs.pop(0) 143 | print 'min', minema, 'max', maxema, 'avg', times[-1] / count, \ 144 | 'ema < half', lthalf * 1.0 / count, \ 145 | 'ema > double', gtdouble * 1.0 / count, \ 146 | 'ema < 3/4', lttq * 1.0 / count, \ 147 | 'ema > 4/3', gtft * 1.0 / count 148 | 149 | # Example usage 150 | # blkdiff.test(blkdiff.hashpower, blkdiff.simple_adjust) 151 | -------------------------------------------------------------------------------- /erasure_code/share.cpp: -------------------------------------------------------------------------------- 1 | #include "share.h" 2 | 3 | // returns a * (x+1) in the Galois field 4 | // (since (x+1) is a primitive root) 5 | static constexpr std::uint8_t tpl(std::uint8_t a) { 6 | return a ^ (a<<1) // a * (x+1) 7 | ^ ((a & (1<<7)) != 0 8 | ? // would overflow (have an x^8 term); reduce by the AES polynomial, 9 | // x^8 + x^4 + x^3 + x + 1 10 | 0b00011011u 11 | : 0 12 | ); 13 | } 14 | 15 | // constexpr functions to compute exp/log 16 | // these are not intended to be fast, but they must be constexpr to populate a 17 | // table at compile time 18 | static constexpr std::uint8_t gexp(unsigned k) { 19 | return k > 0 ? tpl(gexp(k-1)) : 1; 20 | } 21 | static constexpr std::uint8_t glog(unsigned k, unsigned i = 0, unsigned v = 1) { 22 | return k == v ? i : glog(k, i+1, tpl(v)); 23 | } 24 | 25 | // insane hack (courtesy of Xeo on stackoverflow): gen_seq expands to a 26 | // struct that derives from seq<0, 1, ..., N-1> 27 | template struct seq{}; 28 | template 29 | struct gen_seq : gen_seq{}; 30 | template 31 | struct gen_seq<0, I...> : seq{}; 32 | 33 | // produce the actual tables in array form... 34 | template 35 | constexpr std::array exptbl(seq) { 36 | return { { Galois(gexp(I))... } }; 37 | } 38 | template 39 | constexpr std::array logtbl(seq) { 40 | // manually populate entry zero, for two reasons: 41 | // - it makes glog simpler 42 | // - it avoids clang++'s default template instantiation depth limit of 256 43 | return { { 0, glog(I+1)... } }; 44 | } 45 | 46 | // and initialize the static variables 47 | const std::array Galois::exptable = exptbl(gen_seq<255>{}); 48 | const std::array Galois::logtable = logtbl(gen_seq<255>{}); 49 | 50 | // by populating everything at compile-time, we avoid a static initialization 51 | // step and any possible associated static initialization "races" 52 | -------------------------------------------------------------------------------- /erasure_code/share.go: -------------------------------------------------------------------------------- 1 | package erasure_code 2 | 3 | import "fmt" 4 | 5 | func init() { 6 | galoisInit() 7 | } 8 | 9 | // Finite fields 10 | // ============= 11 | 12 | type ZeroDivisionError struct { 13 | } 14 | 15 | func (e *ZeroDivisionError) Error() string { 16 | return "division by zero" 17 | } 18 | 19 | // should panic with ZeroDivisionError when dividing by zero 20 | type Field interface { 21 | Add(b Field) Field 22 | Sub(b Field) Field 23 | Mul(b Field) Field 24 | Div(b Field) Field 25 | Value() int 26 | Factory() FieldFactory 27 | } 28 | type FieldFactory interface { 29 | Construct(v int) Field 30 | } 31 | 32 | // per-byte 2^8 Galois field 33 | // Note that this imposes a hard limit that the number of extended chunks can 34 | // be at most 256 along each dimension 35 | type Galois struct { 36 | v uint8 37 | } 38 | 39 | var gexptable [255]uint8 40 | var glogtable [256]uint8 41 | 42 | func galoisTpl(a uint8) uint8 { 43 | r := a ^ (a << 1) // a * (x+1) 44 | if (a & (1 << 7)) != 0 { 45 | // would overflow (have an x^8 term); reduce by the AES polynomial, 46 | // x^8 + x^4 + x^3 + x + 1 47 | return r ^ 0x1b 48 | } else { 49 | return r 50 | } 51 | } 52 | 53 | func galoisInit() { 54 | var v uint8 = 1 55 | for i := uint8(0); i < 255; i++ { 56 | glogtable[v] = i 57 | gexptable[i] = v 58 | v = galoisTpl(v) 59 | } 60 | } 61 | 62 | func (a *Galois) Add(_b Field) Field { 63 | b := _b.(*Galois) 64 | return &Galois{a.v ^ b.v} 65 | } 66 | func (a *Galois) Sub(_b Field) Field { 67 | b := _b.(*Galois) 68 | return &Galois{a.v ^ b.v} 69 | } 70 | func (a *Galois) Mul(_b Field) Field { 71 | b := _b.(*Galois) 72 | if a.v == 0 || b.v == 0 { 73 | return &Galois{0} 74 | } 75 | return &Galois{gexptable[(int(glogtable[a.v])+ 76 | int(glogtable[b.v]))%255]} 77 | } 78 | func (a *Galois) Div(_b Field) Field { 79 | b := _b.(*Galois) 80 | if b.v == 0 { 81 | panic(ZeroDivisionError{}) 82 | } 83 | if a.v == 0 { 84 | return &Galois{0} 85 | } 86 | return &Galois{gexptable[(int(glogtable[a.v])+255- 87 | int(glogtable[b.v]))%255]} 88 | } 89 | func (a *Galois) Value() int { 90 | return int(a.v) 91 | } 92 | func (a *Galois) String() string { 93 | return fmt.Sprintf("%d",a.v) 94 | } 95 | 96 | type galoisFactory struct { 97 | } 98 | 99 | func GaloisFactory() FieldFactory { 100 | return &galoisFactory{} 101 | } 102 | func (self *Galois) Factory() FieldFactory { 103 | return GaloisFactory() 104 | } 105 | func (self *galoisFactory) Construct(v int) Field { 106 | return &Galois{uint8(v)} 107 | } 108 | 109 | // Modular arithmetic class 110 | type modulo struct { 111 | v uint 112 | n uint // the modulus 113 | } 114 | 115 | func (a *modulo) Add(_b Field) Field { 116 | b := _b.(*modulo) 117 | return &modulo{(a.v + b.v) % a.n, a.n} 118 | } 119 | 120 | func (a *modulo) Sub(_b Field) Field { 121 | b := _b.(*modulo) 122 | return &modulo{(a.v + a.n - b.v) % a.n, a.n} 123 | } 124 | 125 | func (a *modulo) Mul(_b Field) Field { 126 | b := _b.(*modulo) 127 | return &modulo{(a.v * b.v) % a.n, a.n} 128 | } 129 | 130 | func powmod(b uint, e uint, m uint) uint { 131 | var r uint = 1 132 | for e > 0 { 133 | if (e & 1) == 1 { 134 | r = (r * b) % m 135 | } 136 | b = (b * b) % m 137 | e >>= 1 138 | } 139 | return r 140 | } 141 | 142 | func (a *modulo) Div(_b Field) Field { 143 | b := _b.(*modulo) 144 | return &modulo{(a.v * powmod(b.v, a.n-2, a.n)) % a.n, a.n} 145 | } 146 | 147 | func (self *modulo) Value() int { 148 | return int(self.v) 149 | } 150 | 151 | type moduloFactory struct { 152 | n uint 153 | } 154 | 155 | func (self *modulo) Factory() FieldFactory { 156 | return &moduloFactory{self.n} 157 | } 158 | 159 | func (self *moduloFactory) Construct(v int) Field { 160 | return &modulo{uint(v), self.n} 161 | } 162 | 163 | func MakeModuloFactory(n uint) FieldFactory { 164 | return &moduloFactory{n} 165 | } 166 | 167 | func zero(f FieldFactory) Field { 168 | return f.Construct(0) 169 | } 170 | func one(f FieldFactory) Field { 171 | return f.Construct(1) 172 | } 173 | 174 | // Helper functions 175 | // ================ 176 | 177 | // Evaluates a polynomial p in little-endian form (e.g. x^2 + 3x + 2 is 178 | // represented as [2, 3, 1]) at coordinate x, 179 | func EvalPolyAt(poly []Field, x Field) Field { 180 | arithmetic := x.Factory() 181 | r, xi := zero(arithmetic), one(arithmetic) 182 | for _, ci := range poly { 183 | r = r.Add(xi.Mul(ci)) 184 | xi = xi.Mul(x) 185 | } 186 | return r 187 | } 188 | 189 | // Given p+1 y values and x values with no errors, recovers the original 190 | // p+1 degree polynomial. For example, 191 | // LagrangeInterp({51.0, 59.0, 66.0}, {1, 3, 4}) = {50.0, 0, 1.0} 192 | // (or it would be, if floats were Fields) 193 | func LagrangeInterp(pieces []Field, xs []Field) []Field { 194 | arithmetic := pieces[0].Factory() 195 | zero, one := zero(arithmetic), one(arithmetic) 196 | 197 | // `size` is the number of datapoints; the degree of the result polynomial 198 | // is then `size-1` 199 | size := len(pieces) 200 | 201 | root := []Field{one} // initially just the polynomial "1" 202 | // build up the numerator polynomial, `root`, by taking the product of (x-v) 203 | // (implemented as convolving repeatedly with [-v, 1]) 204 | for _, v := range xs { 205 | // iterate backward since new root[i] depends on old root[i-1] 206 | for i := len(root) - 1; i >= 0; i-- { 207 | root[i] = root[i].Mul(zero.Sub(v)) 208 | if i > 0 { 209 | root[i] = root[i].Add(root[i-1]) 210 | } 211 | } 212 | // polynomial is always monic so save an extra multiply by doing this 213 | // after 214 | root = append(root, one) 215 | } 216 | 217 | // generate per-value numerator polynomials by dividing the master 218 | // polynomial back by each x coordinate 219 | nums := make([][]Field, size) 220 | for i, v := range xs { 221 | // divide `root` by (x-v) to get a degree size-1 polynomial 222 | // (i.e. with `size` coefficients) 223 | num := make([]Field, size) 224 | // compute the x^0, x^1, ..., x^(p-2) coefficients by long division 225 | last := one 226 | num[len(num)-1] = last // still always a monic polynomial 227 | for j := size - 2; j >= 0; j-- { 228 | last = root[j+1].Add(last.Mul(v)) 229 | num[j] = last 230 | } 231 | nums[i] = num 232 | } 233 | 234 | // generate denominators by evaluating numerator polys at their x 235 | denoms := make([]Field, size) 236 | for i, x := range xs { 237 | denoms[i] = EvalPolyAt(nums[i], x) 238 | } 239 | 240 | // generate output polynomial by taking the sum over i of 241 | // (nums[i] * pieces[i] / denoms[i]) 242 | sum := make([]Field, size) 243 | for i := range sum { 244 | sum[i] = zero 245 | } 246 | for i, y := range pieces { 247 | factor := y.Div(denoms[i]) 248 | // add nums[i] * factor to sum, as a vector 249 | for j := 0; j < size; j++ { 250 | sum[j] = sum[j].Add(nums[i][j].Mul(factor)) 251 | } 252 | } 253 | return sum 254 | } 255 | 256 | // Given two linear equations, eliminates the first variable and returns 257 | // the resulting equation. 258 | // 259 | // An equation of the form a_1 x_1 + ... + a_n x_n + b = 0 260 | // is represented as the array [a_1, ..., a_n, b]. 261 | func elim(a []Field, b []Field) []Field { 262 | result := make([]Field, len(a)-1) 263 | for i := range result { 264 | result[i] = a[i+1].Mul(b[0]).Sub(b[i+1].Mul(a[0])) 265 | } 266 | return result 267 | } 268 | 269 | // Given one homogeneous linear equation and the values of all but the first 270 | // variable, solve for the value of the first variable. 271 | // 272 | // For an equation of the form 273 | // a_1 x_1 + ... + a_n x_n = 0 274 | // pass two arrays, [a_1, ..., a_n] and [x_2, ..., x_n]. 275 | func evaluate(coeffs []Field, vals []Field) Field { 276 | total := zero(coeffs[0].Factory()) 277 | for i, val := range vals { 278 | total = total.Sub(coeffs[i+1].Mul(val)) 279 | } 280 | return total.Div(coeffs[0]) 281 | } 282 | 283 | // Given an n*n system of inhomogeneous linear equations, solve for the value of 284 | // every variable. 285 | // 286 | // For equations of the form 287 | // a_1,1 x_1 + ... + a_1,n x_n + b_1 = 0 288 | // a_2,1 x_1 + ... + a_2,n x_n + b_2 = 0 289 | // ... 290 | // a_n,1 x_1 + ... + a_n,n x_n + b_n = 0 291 | // pass a two-dimensional array 292 | // [[a_1,1, ..., a_1,n, b_1], ..., [a_n,1, ..., a_n,n, b_n]]. 293 | // 294 | // Returns the values of [x_1, ..., x_n]. 295 | func SysSolve(eqs [][]Field) []Field { 296 | arithmetic := eqs[0][0].Factory() 297 | backEqs := make([][]Field, 1, len(eqs)) 298 | backEqs[0] = eqs[0] 299 | 300 | for len(eqs) > 1 { 301 | neweqs := make([][]Field, len(eqs)-1) 302 | for i := 0; i < len(eqs)-1; i++ { 303 | neweqs[i] = elim(eqs[i], eqs[i+1]) 304 | } 305 | eqs = neweqs 306 | // find a row with a nonzero first entry 307 | i := 0 308 | for i+1 < len(eqs) && eqs[i][0].Value() == 0 { 309 | i++ 310 | } 311 | backEqs = append(backEqs, eqs[i]) 312 | } 313 | 314 | kvals := make([]Field, len(backEqs)+1) 315 | kvals[len(backEqs)] = one(arithmetic) 316 | // back-substitute in reverse order 317 | // (smallest to largest equation) 318 | for i := len(backEqs) - 1; i >= 0; i-- { 319 | kvals[i] = evaluate(backEqs[i], kvals[i+1:]) 320 | } 321 | 322 | return kvals[:len(kvals)-1] 323 | } 324 | 325 | // Divide two polynomials with nonzero leading terms. 326 | // T should be a field. 327 | func PolyDiv(Q []Field, E []Field) []Field { 328 | if len(Q) < len(E) { 329 | return []Field{} 330 | } 331 | div := make([]Field, len(Q)-len(E)+1) 332 | for i := len(div) - 1; i >= 0; i-- { 333 | factor := Q[len(Q)-1].Div(E[len(E)-1]) 334 | div[i] = factor 335 | // subtract factor * E * x^i from Q 336 | Q = Q[:len(Q)-1] // the highest term should cancel 337 | for j := 0; j < len(E)-1; j++ { 338 | Q[i+j] = Q[i+j].Sub(factor.Mul(E[j])) 339 | } 340 | } 341 | return div 342 | } 343 | 344 | func trySysSolve(eqs [][]Field) (ret []Field, ok bool) { 345 | defer func() { 346 | err := recover() 347 | if err == nil { 348 | return 349 | } 350 | switch err := err.(type) { 351 | case ZeroDivisionError: 352 | ret = nil 353 | ok = false 354 | default: 355 | panic(err) 356 | } 357 | }() 358 | return SysSolve(eqs), true 359 | } 360 | 361 | type NotEnoughData struct { } 362 | func (self *NotEnoughData) Error() string { 363 | return "Not enough data!" 364 | } 365 | type TooManyErrors struct { } 366 | func (self *TooManyErrors) Error() string { 367 | return "Answer doesn't match (too many errors)!" 368 | } 369 | 370 | // Given a set of y coordinates and x coordinates, and the degree of the 371 | // original polynomial, determines the original polynomial even if some of the y 372 | // coordinates are wrong. If m is the minimal number of pieces (ie. degree + 373 | // 1), t is the total number of pieces provided, then the algo can handle up to 374 | // (t-m)/2 errors. 375 | func BerlekampWelchAttempt(pieces []Field, xs []Field, masterDegree int) ([]Field, error) { 376 | errorLocatorDegree := (len(pieces) - masterDegree - 1) / 2 377 | arithmetic := pieces[0].Factory() 378 | zero, one := zero(arithmetic), one(arithmetic) 379 | // Set up the equations for y[i]E(x[i]) = Q(x[i]) 380 | // degree(E) = errorLocatorDegree 381 | // degree(Q) = masterDegree + errorLocatorDegree - 1 382 | eqs := make([][]Field, 2*errorLocatorDegree+masterDegree+1) 383 | for i := range eqs { 384 | eq := []Field{} 385 | x := xs[i] 386 | piece := pieces[i] 387 | neg_x_j := zero.Sub(one) 388 | for j := 0; j < errorLocatorDegree+masterDegree+1; j++ { 389 | eq = append(eq, neg_x_j) 390 | neg_x_j = neg_x_j.Mul(x) 391 | } 392 | x_j := one 393 | for j := 0; j < errorLocatorDegree+1; j++ { 394 | eq = append(eq, x_j.Mul(piece)) 395 | x_j = x_j.Mul(x) 396 | } 397 | eqs[i] = eq 398 | } 399 | // Solve the equations 400 | // Assume the top error polynomial term to be one 401 | errors := errorLocatorDegree 402 | ones := 1 403 | var polys []Field 404 | for errors >= 0 { 405 | if p, ok := trySysSolve(eqs); ok { 406 | for i := 0; i < ones; i++ { 407 | p = append(p, one) 408 | } 409 | polys = p 410 | break 411 | } 412 | // caught ZeroDivisionError 413 | eqs = eqs[:len(eqs)-1] 414 | for i, eq := range eqs { 415 | eq[len(eq)-2] = eq[len(eq)-2].Add(eq[len(eq)-1]) 416 | eqs[i] = eq[:len(eq)-1] 417 | } 418 | errors-- 419 | ones++ 420 | } 421 | if errors < 0 { 422 | return nil, &NotEnoughData{} 423 | } 424 | // divide the polynomials... 425 | split := errorLocatorDegree + masterDegree + 1 426 | div := PolyDiv(polys[:split], polys[split:]) 427 | corrects := 0 428 | for i := 0; i < len(xs); i++ { 429 | if EvalPolyAt(div, xs[i]).Value() == pieces[i].Value() { 430 | corrects++ 431 | } 432 | } 433 | if corrects < masterDegree+errors { 434 | return nil, &TooManyErrors{} 435 | } 436 | return div, nil 437 | } 438 | 439 | // Extends a list of integers in [0 ... 255] (if using Galois arithmetic) by 440 | // adding n redundant error-correction values 441 | func Extend(data []int, n int, arithmetic FieldFactory) ([]int, error) { 442 | size := len(data) 443 | 444 | dataF := make([]Field, size) 445 | for i, d := range data { 446 | dataF[i] = arithmetic.Construct(d) 447 | } 448 | 449 | xs := make([]Field, size) 450 | for i := range xs { 451 | xs[i] = arithmetic.Construct(i) 452 | } 453 | 454 | poly, err := BerlekampWelchAttempt(dataF, xs, size-1) 455 | if err != nil { 456 | return nil, err 457 | } 458 | 459 | for i := 0; i < n; i++ { 460 | data = append(data, 461 | int(EvalPolyAt(poly, arithmetic.Construct(size+i)).Value())) 462 | } 463 | return data, nil 464 | } 465 | 466 | // Repairs a list of integers in [0 ... 255]. Some integers can be erroneous, 467 | // and you can put -1 in place of an integer if you know that a certain 468 | // value is defective or missing. Uses the Berlekamp-Welch algorithm to 469 | // do error-correction 470 | func Repair(data []int, datasize int, arithmetic FieldFactory) ([]int, error) { 471 | vs := make([]Field, 0, len(data)) 472 | xs := make([]Field, 0, len(data)) 473 | for i, d := range data { 474 | if d >= 0 { 475 | vs = append(vs, arithmetic.Construct(d)) 476 | xs = append(xs, arithmetic.Construct(i)) 477 | } 478 | } 479 | 480 | poly, err := BerlekampWelchAttempt(vs, xs, datasize-1) 481 | if err != nil { 482 | return nil, err 483 | } 484 | 485 | result := make([]int, len(data)) 486 | for i := range result { 487 | result[i] = int(EvalPolyAt(poly, arithmetic.Construct(i)).Value()) 488 | } 489 | return result, nil 490 | } 491 | 492 | func transpose(d [][]int) [][]int { 493 | width := len(d[0]) 494 | result := make([][]int, width) 495 | for i := range result { 496 | col := make([]int, len(d)) 497 | for j := range col { 498 | col[j] = d[j][i] 499 | } 500 | result[i] = col 501 | } 502 | return result 503 | } 504 | 505 | func extractColumn(d [][]int, j int) []int { 506 | result := make([]int, len(d)) 507 | for i, row := range d { 508 | result[i] = row[j] 509 | } 510 | return result 511 | } 512 | 513 | // Extends a list of bytearrays 514 | // eg. ExtendChunks([map(ord, 'hello'), map(ord, 'world')], 2) 515 | // n is the number of redundant error-correction chunks to add 516 | func ExtendChunks(data [][]int, n int, arithmetic FieldFactory) ([][]int, error) { 517 | width := len(data[0]) 518 | o := make([][]int, width) 519 | for i := 0; i < width; i++ { 520 | row, err := Extend(extractColumn(data, i), n, arithmetic) 521 | if err != nil { 522 | return nil, err 523 | } 524 | o[i] = row 525 | } 526 | return transpose(o), nil 527 | } 528 | 529 | // Repairs a list of bytearrays. Use an empty array in place of a missing array. 530 | // Individual arrays can contain some missing or erroneous data. 531 | func RepairChunks(data [][]int, datasize int, arithmetic FieldFactory) ([][]int, error) { 532 | var width int 533 | for _, row := range data { 534 | if len(row) > 0 { 535 | width = len(row) 536 | break 537 | } 538 | } 539 | filledData := make([][]int, len(data)) 540 | for i, row := range data { 541 | if len(row) == 0 { 542 | filledData[i] = make([]int, width) 543 | for j := range filledData[i] { 544 | filledData[i][j] = -1 545 | } 546 | } else { 547 | filledData[i] = row 548 | } 549 | } 550 | o := make([][]int, width) 551 | for i := range o { 552 | row, err := Repair(extractColumn(data, i), datasize, arithmetic) 553 | if err != nil { 554 | return nil, err 555 | } 556 | o[i] = row 557 | } 558 | return transpose(o), nil 559 | } 560 | -------------------------------------------------------------------------------- /erasure_code/share.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "utils.h" 9 | 10 | class ZeroDivisionError : std::domain_error { 11 | public: 12 | ZeroDivisionError() : domain_error("division by zero") { } 13 | }; 14 | 15 | // GF(2^8) in the form (Z/2Z)[x]/(x^8+x^4+x^3+x+1) 16 | // (the AES polynomial) 17 | class Galois { 18 | // the coefficients of the polynomial, where the ith bit of `val` is the x^i 19 | // coefficient 20 | std::uint8_t v; 21 | 22 | // precomputed data: log and exp tables 23 | static const std::array exptable; 24 | static const std::array logtable; 25 | 26 | public: 27 | explicit constexpr Galois(unsigned char val) : v(val) { } 28 | 29 | Galois operator+(Galois b) const { 30 | return Galois(v ^ b.v); 31 | } 32 | Galois operator-(Galois b) const { 33 | return Galois(v ^ b.v); 34 | } 35 | Galois operator*(Galois b) const { 36 | return v == 0 || b.v == 0 37 | ? Galois(0) 38 | : exptable[(unsigned(logtable[v]) + logtable[b.v]) % 255]; 39 | } 40 | Galois operator/(Galois b) const { 41 | if (b.v == 0) { 42 | throw ZeroDivisionError(); 43 | } 44 | return v == 0 || b.v == 0 45 | ? Galois(0) 46 | : exptable[(unsigned(logtable[v]) + 255u - logtable[b.v]) % 255]; 47 | } 48 | Galois operator-() const { 49 | return *this; 50 | } 51 | 52 | Galois& operator+=(Galois b) { 53 | return *this = *this + b; 54 | } 55 | Galois& operator-=(Galois b) { 56 | return *this = *this - b; 57 | } 58 | Galois& operator*=(Galois b) { 59 | return *this = *this * b; 60 | } 61 | Galois& operator/=(Galois b) { 62 | return *this = *this / b; 63 | } 64 | 65 | bool operator==(Galois b) { 66 | return v == b.v; 67 | } 68 | 69 | // back door 70 | std::uint8_t val() const { 71 | return v; 72 | } 73 | }; 74 | 75 | // Z/pZ, for an odd prime p 76 | template 77 | class Modulo { 78 | // check that p is prime by trial division 79 | static constexpr bool is_prime(unsigned x, unsigned divisor = 2) { 80 | return divisor * divisor > x 81 | ? true 82 | : x % divisor != 0 && is_prime(x, divisor + 1); 83 | } 84 | static_assert(p > 2 && is_prime(p, 2), "p must be an odd prime!"); 85 | 86 | unsigned v; 87 | 88 | public: 89 | explicit Modulo(unsigned val) : v(val) { 90 | assert(v >= 0 && v < p); 91 | } 92 | 93 | 94 | Modulo inv() const { 95 | if (v == 0) { 96 | throw ZeroDivisionError(); 97 | } 98 | unsigned r = 1, base = v, exp = p-2; 99 | while (exp > 0) { 100 | if (exp & 1) r = (r * base) % p; 101 | base = (base * base) % p; 102 | exp >>= 1; 103 | } 104 | return Modulo(r); 105 | } 106 | Modulo operator+(Modulo b) const { 107 | return Modulo((v + b.v) % p); 108 | } 109 | Modulo operator-(Modulo b) const { 110 | return Modulo((v + p - b.v) % p); 111 | } 112 | Modulo operator*(Modulo b) const { 113 | return Modulo((v * b.v) % p); 114 | } 115 | Modulo operator/(Modulo b) const { 116 | return *this * b.inv(); 117 | } 118 | 119 | Modulo& operator+=(Modulo b) { 120 | return *this = *this + b; 121 | } 122 | Modulo& operator-=(Modulo b) { 123 | return *this = *this - b; 124 | } 125 | Modulo& operator*=(Modulo b) { 126 | return *this = *this * b; 127 | } 128 | Modulo& operator/=(Modulo b) { 129 | return *this = *this / b; 130 | } 131 | 132 | bool operator==(Modulo b) { 133 | return v == b.v; 134 | } 135 | 136 | // back door 137 | unsigned val() const { 138 | return v; 139 | } 140 | }; 141 | 142 | // Evaluates a polynomial p in little-endian form (e.g. x^2 + 3x + 2 is 143 | // represented as {2, 3, 1}) at coordinate x, 144 | // e.g. eval_poly_at((int[]){2, 3, 1}, 5) = 42. 145 | // 146 | // T should be a type supporting ring arithmetic and T(0) and T(1) should be the 147 | // appropriate identities. 148 | // 149 | // Range should be a type that can be iterated to get const T& elements. 150 | template 151 | T eval_poly_at(const Range& p, T x) { 152 | T r(0), xi(1); 153 | for (const T& c_i : p) { 154 | r += c_i * xi; 155 | xi *= x; 156 | } 157 | return r; 158 | } 159 | 160 | // Given p+1 y values and x values with no errors, recovers the original 161 | // degree-p polynomial. For example, 162 | // lagrange_interp((double[]){51.0, 59.0, 66.0}, 163 | // (double[]){1.0, 3.0, 4.0}) 164 | // = {50.0, 0.0, 1.0}. 165 | // 166 | // T should be a field and Range should be a sized range type with values of 167 | // type T. T(0) and T(1) should be the appropriate field identities. 168 | template 169 | std::vector lagrange_interp(const Range& pieces, const Range& xs) { 170 | // `size` is the number of datapoints; the degree of the result polynomial 171 | // is then `size-1` 172 | const unsigned size = pieces.size(); 173 | assert(size == xs.size()); 174 | 175 | std::vector root{T(1)}; // initially just the polynomial "1" 176 | // build up the numerator polynomial, `root`, by taking the product of (x-v) 177 | // (implemented as convolving repeatedly with [-v, 1]) 178 | for (const T& v : xs) { 179 | // iterate backward since new root[i] depends on old root[i-1] 180 | for (unsigned i = root.size(); i--; ) { 181 | root[i] *= -v; 182 | if (i > 0) root[i] += root[i-1]; 183 | } 184 | // polynomial is always monic so save an extra multiply by doing this 185 | // after 186 | root.emplace_back(1); 187 | } 188 | // should have degree `size` 189 | assert(root.size() == size + 1); 190 | 191 | // generate per-value numerator polynomials by dividing the master 192 | // polynomial back by each x coordinate 193 | std::vector > nums; 194 | nums.reserve(size); 195 | for (const T& v : xs) { 196 | // divide `root` by (x-v) to get a degree size-1 polynomial 197 | // (i.e. with `size` coefficients) 198 | std::vector num(size, T(0)); 199 | // compute the x^0, x^1, ..., x^(p-2) coefficients by long division 200 | T last = num.back() = T(1); // still always a monic polynomial 201 | for (int i = int(size)-2; i >= 0; --i) { 202 | num[i] = last = root[i+1] + last * v; 203 | } 204 | nums.emplace_back(std::move(num)); 205 | } 206 | assert(nums.size() == size); 207 | 208 | // generate denominators by evaluating numerator polys at their x 209 | std::vector denoms; 210 | denoms.reserve(size); 211 | { 212 | unsigned i = 0; 213 | for (const T& v : xs) { 214 | denoms.push_back(eval_poly_at(nums[i], v)); 215 | ++i; 216 | } 217 | } 218 | assert(denoms.size() == size); 219 | 220 | // generate output polynomial by taking the sum over i of 221 | // (nums[i] * pieces[i] / denoms[i]) 222 | std::vector sum(size, T(0)); 223 | { 224 | unsigned i = 0; 225 | for (const T& y : pieces) { 226 | T factor = y / denoms[i]; 227 | // add nums[i] * factor to sum, as a vector 228 | for (unsigned j = 0; j < size; ++j) { 229 | sum[j] += nums[i][j] * factor; 230 | } 231 | ++i; 232 | } 233 | } 234 | return sum; 235 | } 236 | 237 | // Given two linear equations, eliminates the first variable and returns 238 | // the resulting equation. 239 | // 240 | // An equation of the form a_1 x_1 + ... + a_n x_n + b = 0 241 | // is represented as the array [a_1, ..., a_n, b]. 242 | // 243 | // T should be a ring and Range should be an indexable, sized range of T. 244 | template 245 | std::vector elim(const Range& a, const Range& b) { 246 | assert(a.size() == b.size()); 247 | std::vector result; 248 | const unsigned size = a.size(); 249 | for (unsigned i = 1; i < size; ++i) { 250 | result.push_back(a[i] * b[0] - b[i] * a[0]); 251 | } 252 | return result; 253 | } 254 | 255 | // Given one homogeneous linear equation and the values of all but the first 256 | // variable, solve for the value of the first variable. 257 | // 258 | // For an equation of the form 259 | // a_1 x_1 + ... + a_n x_n = 0 260 | // pass two arrays, [a_1, ..., a_n] and [x_2, ..., x_n]. 261 | // 262 | // T should be a field; and R1 and R2 should be indexable, sized ranges of T. 263 | template 264 | T evaluate(const R1& coeffs, const R2& vals) { 265 | assert(coeffs.size() == vals.size() + 1); 266 | T total(0); 267 | const unsigned size = vals.size(); 268 | for (unsigned i = 0; i < size; ++i) { 269 | total -= coeffs[i+1] * vals[i]; 270 | } 271 | return total / coeffs[0]; 272 | } 273 | 274 | // Given an n*n system of inhomogeneous linear equations, solve for the value of 275 | // every variable. 276 | // 277 | // For equations of the form 278 | // a_1,1 x_1 + ... + a_1,n x_n + b_1 = 0 279 | // a_2,1 x_1 + ... + a_2,n x_n + b_2 = 0 280 | // ... 281 | // a_n,1 x_1 + ... + a_n,n x_n + b_n = 0 282 | // pass a two-dimensional array 283 | // [[a_1,1, ..., a_1,n, b_1], ..., [a_n,1, ..., a_n,n, b_n]]. 284 | // 285 | // Returns the values of [x_1, ..., x_n]. 286 | // 287 | // T should be a field. 288 | template 289 | std::vector sys_solve(std::vector> eqs) { 290 | assert(eqs.size() > 0); 291 | std::vector> back_eqs{eqs[0]}; 292 | 293 | while (eqs.size() > 1) { 294 | std::vector> neweqs; 295 | neweqs.reserve(eqs.size()-1); 296 | for (unsigned i = 0; i < eqs.size()-1; ++i) { 297 | neweqs.push_back(elim(eqs[i], eqs[i+1])); 298 | } 299 | eqs = std::move(neweqs); 300 | // find a row with a nonzero first entry 301 | unsigned i = 0; 302 | while (i + 1 < eqs.size() && eqs[i][0] == T(0)) { 303 | ++i; 304 | } 305 | back_eqs.push_back(eqs[i]); 306 | } 307 | 308 | std::vector kvals(back_eqs.size()+1, T(0)); 309 | kvals.back() = T(1); 310 | // back-substitute in reverse order 311 | // (smallest to largest equation) 312 | for (unsigned i = back_eqs.size(); i--; ) { 313 | kvals[i] = evaluate(back_eqs[i], 314 | // use the already-computed values + the 1 at the end 315 | make_iter_pair(kvals.begin()+i+1, kvals.end())); 316 | } 317 | 318 | kvals.pop_back(); 319 | 320 | return kvals; 321 | } 322 | 323 | // Divide two polynomials with nonzero leading terms. 324 | // T should be a field. 325 | template 326 | std::vector polydiv(std::vector Q, const std::vector& E) { 327 | if (Q.size() < E.size()) return {}; 328 | std::vector div(Q.size() - E.size() + 1, T(0)); 329 | unsigned i = div.size(); 330 | while (i--) { 331 | T factor = Q.back() / E.back(); 332 | div[i] = factor; 333 | // subtract factor * E * x^i from Q 334 | Q.pop_back(); // the highest term should cancel 335 | for (unsigned j = 0; j < E.size() - 1; ++j) { 336 | Q[i+j] -= factor * E[j]; 337 | } 338 | assert(Q.size() == i + E.size() - 1); 339 | } 340 | return div; 341 | } 342 | 343 | // Given a set of y coordinates and x coordinates, and the degree of the 344 | // original polynomial, determines the original polynomial even if some of the y 345 | // coordinates are wrong. If m is the minimal number of pieces (ie. degree + 346 | // 1), t is the total number of pieces provided, then the algo can handle up to 347 | // (t-m)/2 errors. 348 | // 349 | // T should be a field. In particular, division by zero over T should throw 350 | // ZeroDivisionError. 351 | template 352 | std::vector berlekamp_welch_attempt(const std::vector& pieces, 353 | const std::vector& xs, unsigned master_degree) { 354 | const unsigned error_locator_degree = (pieces.size() - master_degree - 1) / 2; 355 | // Set up the equations for y[i]E(x[i]) = Q(x[i]) 356 | // degree(E) = error_locator_degree 357 | // degree(Q) = master_degree + error_locator_degree - 1 358 | std::vector> eqs(2*error_locator_degree + master_degree + 1); 359 | for (unsigned i = 0; i < eqs.size(); ++i) { 360 | std::vector& eq = eqs[i]; 361 | const T& x = xs[i]; 362 | const T& piece = pieces[i]; 363 | T neg_x_j = T(0) - T(1); 364 | for (unsigned j = 0; j < error_locator_degree + master_degree + 1; ++j) { 365 | eq.push_back(neg_x_j); 366 | neg_x_j *= x; 367 | } 368 | T x_j = T(1); 369 | for (unsigned j = 0; j < error_locator_degree + 1; ++j) { 370 | eq.push_back(x_j * piece); 371 | x_j *= x; 372 | } 373 | } 374 | // Solve the equations 375 | // Assume the top error polynomial term to be one 376 | int errors = error_locator_degree; 377 | unsigned ones = 1; 378 | std::vector polys; 379 | while (errors >= 0) { 380 | try { 381 | polys = sys_solve(eqs); 382 | } catch (const ZeroDivisionError&) { 383 | eqs.pop_back(); 384 | for (auto& eq : eqs) { 385 | eq[eq.size()-2] += eq.back(); 386 | eq.pop_back(); 387 | } 388 | --errors; 389 | ++ones; 390 | continue; 391 | } 392 | for (unsigned i = 0; i < ones; ++i) polys.emplace_back(1); 393 | break; 394 | } 395 | if (errors < 0) { 396 | throw std::logic_error("Not enough data!"); 397 | } 398 | // divide the polynomials... 399 | const unsigned split = error_locator_degree + master_degree + 1; 400 | std::vector div = polydiv(std::vector(polys.begin(), polys.begin() + split), 401 | std::vector(polys.begin() + split, polys.end())); 402 | unsigned corrects = 0; 403 | for (unsigned i = 0; i < xs.size(); ++i) { 404 | if (eval_poly_at(div, xs[i]) == pieces[i]) { 405 | ++corrects; 406 | } 407 | } 408 | if (corrects < master_degree + errors) { 409 | throw std::logic_error("Answer doesn't match (too many errors)!"); 410 | } 411 | return div; 412 | } 413 | 414 | // Extends a list of integers in [0 ... 255] (if using Galois arithmetic) by 415 | // adding n redundant error-correction values 416 | template 417 | std::vector extend(std::vector data, unsigned n) { 418 | const unsigned size = data.size(); 419 | 420 | std::vector data_f; 421 | data_f.reserve(size); 422 | for (T d : data) data_f.emplace_back(d); 423 | 424 | std::vector xs; 425 | for (unsigned i = 0; i < size; ++i) xs.emplace_back(i); 426 | 427 | std::vector poly = berlekamp_welch_attempt(data_f, xs, size-1); 428 | 429 | data.reserve(size+n); 430 | for (unsigned i = 0; i < n; ++i) { 431 | data.push_back(eval_poly_at(poly, F(T(size + i))).val()); 432 | } 433 | return data; 434 | } 435 | 436 | // Repairs a list of integers in [0 ... 255]. Some integers can be erroneous, 437 | // and you can put -1 in place of an integer if you know that a certain 438 | // value is defective or missing. Uses the Berlekamp-Welch algorithm to 439 | // do error-correction 440 | template 441 | std::vector repair(const std::vector& data, unsigned datasize) { 442 | std::vector vs, xs; 443 | for (unsigned i = 0; i < data.size(); ++i) { 444 | if (data[i] >= 0) { 445 | vs.emplace_back(data[i]); 446 | xs.emplace_back(T(i)); 447 | } 448 | } 449 | std::vector poly = berlekamp_welch_attempt(vs, xs, datasize - 1); 450 | std::vector result; 451 | for (unsigned i = 0; i < data.size(); ++i) { 452 | result.push_back(eval_poly_at(poly, F(T(i))).val()); 453 | } 454 | return result; 455 | } 456 | 457 | 458 | template 459 | std::vector> transpose(const std::vector>& d) { 460 | assert(d.size() > 0); 461 | unsigned width = d[0].size(); 462 | std::vector> result(width); 463 | for (unsigned i = 0; i < width; ++i) { 464 | for (unsigned j = 0; j < d.size(); ++j) { 465 | result[i].push_back(d[j][i]); 466 | } 467 | } 468 | return result; 469 | } 470 | 471 | template 472 | std::vector extract_column(const std::vector>& d, unsigned i) { 473 | std::vector result; 474 | for (unsigned j = 0; j < d.size(); ++j) { 475 | result.push_back(d[j][i]); 476 | } 477 | return result; 478 | } 479 | 480 | // Extends a list of bytearrays 481 | // eg. extend_chunks([map(ord, 'hello'), map(ord, 'world')], 2) 482 | // n is the number of redundant error-correction chunks to add 483 | template 484 | std::vector> extend_chunks( 485 | const std::vector>& data, 486 | unsigned n) { 487 | std::vector> o; 488 | const unsigned height = data.size(); 489 | assert(height > 0); 490 | const unsigned width = data[0].size(); 491 | for (unsigned i = 0; i < width; ++i) { 492 | o.push_back(extend(extract_column(data, i), n)); 493 | } 494 | return transpose(o); 495 | } 496 | 497 | // Repairs a list of bytearrays. Use an empty array in place of a missing array. 498 | // Individual arrays can contain some missing or erroneous data. 499 | template 500 | std::vector> repair_chunks( 501 | std::vector> data, 502 | unsigned datasize) { 503 | unsigned width = 0; 504 | for (const std::vector& row : data) { 505 | if (row.size() > 0) { 506 | width = row.size(); 507 | break; 508 | } 509 | } 510 | assert(width > 0); 511 | for (std::vector& row : data) { 512 | if (row.size() == 0) { 513 | row.assign(width, -1); 514 | } else { 515 | assert(row.size() == width); 516 | } 517 | } 518 | std::vector> o; 519 | for (unsigned i = 0; i < width; ++i) { 520 | o.push_back(repair(extract_column(data, i), datasize)); 521 | } 522 | return transpose(o); 523 | } 524 | 525 | // Extends either a bytearray or a list of bytearrays or a list of lists... 526 | // Used in the cubify method to expand a cube in all dimensions 527 | template 528 | struct deep_extend_chunks_helper { 529 | static std::vector go(const std::vector& data, unsigned n) { 530 | return extend(data, n); 531 | } 532 | }; 533 | template 534 | struct deep_extend_chunks_helper, F> { 535 | static std::vector> go(const std::vector>& data, unsigned n) { 536 | std::vector> o; 537 | const unsigned height = data.size(); 538 | assert(height > 0); 539 | const unsigned width = data[0].size(); 540 | for (unsigned i = 0; i < width; ++i) { 541 | o.push_back(deep_extend_chunks_helper::go(extract_column(data, i), n)); 542 | } 543 | return transpose(o); 544 | } 545 | }; 546 | template 547 | std::vector deep_extend_chunks(const std::vector& data, unsigned n) { 548 | return deep_extend_chunks_helper::go(data, n); 549 | } 550 | -------------------------------------------------------------------------------- /erasure_code/share.js: -------------------------------------------------------------------------------- 1 | (function() { 2 | var me = {}; 3 | 4 | function ZeroDivisionError() { 5 | if (!this) return new ZeroDivisionError(); 6 | this.message = "division by zero"; 7 | this.name = "ZeroDivisionError"; 8 | } 9 | me.ZeroDivisionError = ZeroDivisionError; 10 | 11 | // per-byte 2^8 Galois field 12 | // Note that this imposes a hard limit that the number of extended chunks can 13 | // be at most 256 along each dimension 14 | function galoistpl(a) { 15 | // 2 is not a primitive root, so we have to use 3 as our logarithm base 16 | var r = a ^ (a<<1); // a * (x+1) 17 | if (r > 0xff) { // overflow? 18 | r = r ^ 0x11b; 19 | } 20 | return r; 21 | } 22 | 23 | // Precomputing a multiplication and XOR table for increased speed 24 | var glogtable = new Array(256); 25 | var gexptable = []; 26 | (function() { 27 | var v = 1; 28 | for (var i = 0; i < 255; i++) { 29 | glogtable[v] = i; 30 | gexptable.push(v); 31 | v = galoistpl(v); 32 | } 33 | })(); 34 | me.glogtable = glogtable; 35 | me.gexptable = gexptable; 36 | 37 | function Galois(val) { 38 | if (!(this instanceof Galois)) return new Galois(val); 39 | if (val instanceof Galois) { 40 | this.val = val.val; 41 | } else { 42 | this.val = val; 43 | } 44 | if (typeof Object.freeze == 'function') { 45 | Object.freeze(this); 46 | } 47 | } 48 | me.Galois = Galois; 49 | Galois.prototype.add = Galois.prototype.sub = function(other) { 50 | return new Galois(this.val ^ other.val); 51 | }; 52 | Galois.prototype.mul = function(other) { 53 | if (this.val == 0 || other.val == 0) { 54 | return new Galois(0); 55 | } 56 | return new Galois(gexptable[(glogtable[this.val] + 57 | glogtable[other.val]) % 255]); 58 | }; 59 | Galois.prototype.div = function(other) { 60 | if (other.val == 0) { 61 | throw new ZeroDivisionError(); 62 | } 63 | if (this.val == 0) { 64 | return new Galois(0); 65 | } 66 | return new Galois(gexptable[(glogtable[this.val] + 255 - 67 | glogtable[other.val]) % 255]); 68 | }; 69 | Galois.prototype.inspect = function() { 70 | return ""+this.val; 71 | }; 72 | 73 | function powmod(b, e, m) { 74 | var r = 1; 75 | while (e > 0) { 76 | if (e & 1) r = (r * b) % m; 77 | b = (b * b) % m; 78 | e = e >> 1; 79 | } 80 | return r; 81 | } 82 | 83 | 84 | // Modular division class 85 | function mkModuloClass(n) { 86 | if (n <= 2) throw new Error("n must be prime!"); 87 | for (var divisor = 2; divisor * divisor <= n; divisor++) { 88 | if (n % divisor == 0) { 89 | throw new Error("n must be prime!"); 90 | } 91 | } 92 | 93 | function Mod(val) { 94 | if (!(this instanceof Mod)) return new Mod(val); 95 | if (val instanceof Mod) { 96 | this.val = val.val; 97 | } else { 98 | this.val = val; 99 | } 100 | if (typeof Object.freeze == 'function') { 101 | Object.freeze(this); 102 | } 103 | } 104 | Mod.modulo = n; 105 | Mod.prototype.add = function(other) { 106 | return new Mod((this.val + other.val) % n); 107 | }; 108 | Mod.prototype.sub = function(other) { 109 | return new Mod((this.val + n - other.val) % n); 110 | }; 111 | Mod.prototype.mul = function(other) { 112 | return new Mod((this.val * other.val) % n); 113 | }; 114 | Mod.prototype.div = function(other) { 115 | return new Mod((this.val * powmod(other.val, n-2, n)) % n); 116 | }; 117 | Mod.prototype.inspect = function() { 118 | return ""+this.val; 119 | }; 120 | 121 | return Mod; 122 | } 123 | me.mkModuloClass = mkModuloClass; 124 | 125 | // Evaluates a polynomial in little-endian form, eg. x^2 + 3x + 2 = [2, 3, 1] 126 | // (normally I hate little-endian, but in this case dealing with polynomials 127 | // it's justified, since you get the nice property that p[n] is the nth degree 128 | // term of p) at coordinate x, eg. eval_poly_at([2, 3, 1], 5) = 42 if you are 129 | // using float as your arithmetic 130 | function eval_poly_at(p, x) { 131 | var arithmetic = p[0].constructor; 132 | var y = new arithmetic(0); 133 | var x_to_the_i = new arithmetic(1); 134 | for (var i = 0; i < p.length; i++) { 135 | y = y.add(x_to_the_i.mul(p[i])) 136 | x_to_the_i = x_to_the_i.mul(x); 137 | } 138 | return y; 139 | } 140 | me.eval_poly_at = eval_poly_at; 141 | 142 | // Given p+1 y values and x values with no errors, recovers the original 143 | // p+1 degree polynomial. For example, 144 | // lagrange_interp([51.0, 59.0, 66.0], [1, 3, 4]) = [50.0, 0, 1.0] 145 | // if you are using float as your arithmetic 146 | function lagrange_interp(pieces, xs) { 147 | var arithmetic = pieces[0].constructor; 148 | var zero = new arithmetic(0); 149 | var one = new arithmetic(1); 150 | // Generate master numerator polynomial 151 | var root = [one]; 152 | var i, j; 153 | for (i = 0; i < xs.length; i++) { 154 | root.unshift(zero); 155 | for (j = 0; j < root.length - 1; j++) { 156 | root[j] = root[j].sub(root[j+1].mul(xs[i])); 157 | } 158 | } 159 | // Generate per-value numerator polynomials by dividing the master 160 | // polynomial back by each x coordinate 161 | var nums = []; 162 | for (i = 0; i < xs.length; i++) { 163 | var output = []; 164 | var last = one; 165 | for (j = 2; j < root.length+1; j++) { 166 | output.unshift(last); 167 | if (j != root.length) { 168 | last = root[root.length-j].add(last.mul(xs[i])); 169 | } 170 | } 171 | nums.push(output); 172 | } 173 | // Generate denominators by evaluating numerator polys at their x 174 | var denoms = []; 175 | for (i = 0; i < xs.length; i++) { 176 | var denom = zero; 177 | var x_to_the_j = one; 178 | for (j = 0; j < nums[i].length; j++) { 179 | denom = denom.add(x_to_the_j.mul(nums[i][j])); 180 | x_to_the_j = x_to_the_j.mul(xs[i]); 181 | } 182 | denoms.push(denom); 183 | } 184 | // Generate output polynomial 185 | var b = []; 186 | for (i = 0; i < pieces.length; i++) { 187 | b[i] = zero; 188 | } 189 | for (i = 0; i < xs.length; i++) { 190 | var yslice = pieces[i].div(denoms[i]); 191 | for (j = 0; j < pieces.length; j++) { 192 | b[j] = b[j].add(nums[i][j].mul(yslice)); 193 | } 194 | } 195 | return b; 196 | } 197 | me.lagrange_interp = lagrange_interp; 198 | 199 | // Compresses two linear equations of length n into one 200 | // equation of length n-1 201 | // Format: 202 | // 3x + 4y = 80 (ie. 3x + 4y - 80 = 0) -> a = [3,4,-80] 203 | // 5x + 2y = 70 (ie. 5x + 2y - 70 = 0) -> b = [5,2,-70] 204 | function elim(a, b) { 205 | var c = []; 206 | for (var i = 1; i < a.length; i++) { 207 | c[i-1] = a[i].mul(b[0]).sub(b[i].mul(a[0])); 208 | } 209 | return c; 210 | } 211 | 212 | // Linear equation solver 213 | // Format: 214 | // 3x + 4y = 80, y = 5 (ie. 3x + 4y - 80z = 0, y = 5, z = 1) 215 | // -> coeffs = [3,4,-80], vals = [5,1] 216 | function evaluate(coeffs, vals) { 217 | var arithmetic = coeffs[0].constructor; 218 | var tot = new arithmetic(0); 219 | for (var i = 0; i < vals.length; i++) { 220 | tot = tot.sub(coeffs[i+1].mul(vals[i])); 221 | } 222 | if (coeffs[0].val == 0) { 223 | throw new ZeroDivisionError(); 224 | } 225 | return tot.div(coeffs[0]); 226 | } 227 | 228 | // Linear equation system solver 229 | // Format: 230 | // ax + by + c = 0, dx + ey + f = 0 231 | // -> [[a, b, c], [d, e, f]] 232 | // eg. 233 | // [[3.0, 5.0, -13.0], [9.0, 1.0, -11.0]] -> [1.0, 2.0] 234 | function sys_solve(eqs) { 235 | var arithmetic = eqs[0][0].constructor; 236 | var one = new arithmetic(1); 237 | var back_eqs = [eqs[0]]; 238 | var i; 239 | while (eqs.length > 1) { 240 | var neweqs = []; 241 | for (i = 0; i < eqs.length - 1; i++) { 242 | neweqs.push(elim(eqs[i], eqs[i+1])); 243 | } 244 | eqs = neweqs; 245 | i = 0; 246 | while (i < eqs.length - 1 && eqs[i][0].val == 0) { 247 | i++; 248 | } 249 | back_eqs.unshift(eqs[i]); 250 | } 251 | var kvals = [one]; 252 | for (i = 0; i < back_eqs.length; i++) { 253 | kvals.unshift(evaluate(back_eqs[i], kvals)); 254 | } 255 | return kvals.slice(0, -1); 256 | } 257 | me.sys_solve = sys_solve; 258 | 259 | function polydiv(Q, E) { 260 | var qpoly = Q.slice(); 261 | var epoly = E.slice(); 262 | var div = []; 263 | while (qpoly.length >= epoly.length) { 264 | div.unshift(qpoly[qpoly.length-1].div(epoly[epoly.length-1])); 265 | for (var i = 2; i < epoly.length + 1; i++) { 266 | qpoly[qpoly.length-i] = 267 | qpoly[qpoly.length-i].sub(div[0].mul(epoly[epoly.length-i])); 268 | } 269 | qpoly.pop(); 270 | } 271 | return div; 272 | } 273 | me.polydiv = polydiv; 274 | 275 | // Given a set of y coordinates and x coordinates, and the degree of the 276 | // original polynomial, determines the original polynomial even if some of 277 | // the y coordinates are wrong. If m is the minimal number of pieces (ie. 278 | // degree + 1), t is the total number of pieces provided, then the algo can 279 | // handle up to (t-m)/2 errors. See: 280 | // http://en.wikipedia.org/wiki/Berlekamp%E2%80%93Welch_algorithm#Example 281 | // (just skip to my example, the rest of the article sucks imo) 282 | function berlekamp_welch_attempt(pieces, xs, master_degree) { 283 | var error_locator_degree = Math.floor((pieces.length - master_degree - 1) / 2); 284 | var arithmetic = pieces[0].constructor; 285 | var zero = new arithmetic(0); 286 | var one = new arithmetic(1); 287 | // Set up the equations for y[i]E(x[i]) = Q(x[i]) 288 | // degree(E) = error_locator_degree 289 | // degree(Q) = master_degree + error_locator_degree - 1 290 | var eqs = []; 291 | var i,j; 292 | for (i = 0; i < 2 * error_locator_degree + master_degree + 1; i++) { 293 | eqs.push([]); 294 | } 295 | for (i = 0; i < 2 * error_locator_degree + master_degree + 1; i++) { 296 | var neg_x_to_the_j = zero.sub(one); 297 | for (j = 0; j < error_locator_degree + master_degree + 1; j++) { 298 | eqs[i].push(neg_x_to_the_j); 299 | neg_x_to_the_j = neg_x_to_the_j.mul(xs[i]); 300 | } 301 | var x_to_the_j = one; 302 | for (j = 0; j < error_locator_degree + 1; j++) { 303 | eqs[i].push(x_to_the_j.mul(pieces[i])); 304 | x_to_the_j = x_to_the_j.mul(xs[i]); 305 | } 306 | } 307 | // Solve 'em 308 | // Assume the top error polynomial term to be one 309 | var errors = error_locator_degree; 310 | var ones = 1; 311 | var polys; 312 | while (errors >= 0) { 313 | try { 314 | polys = sys_solve(eqs); 315 | for (i = 0; i < ones; i++) polys.push(one); 316 | break; 317 | } catch (e) { 318 | if (e instanceof ZeroDivisionError) { 319 | eqs.pop(); 320 | for (i = 0; i < eqs.length; i++) { 321 | var eq = eqs[i]; 322 | eq[eq.length-2] = eq[eq.length-2].add(eq[eq.length-1]); 323 | eq.pop(); 324 | } 325 | errors--; 326 | ones++; 327 | } else { 328 | throw e; 329 | } 330 | } 331 | } 332 | if (errors < 0) { 333 | throw new Error("Not enough data!"); 334 | } 335 | // Divide the polynomials 336 | var qpoly = polys.slice(0, error_locator_degree + master_degree + 1); 337 | var epoly = polys.slice(error_locator_degree + master_degree + 1); 338 | var div = polydiv(qpoly, epoly); 339 | // Check 340 | var corrects = 0; 341 | for (i = 0; i < xs.length; i++) { 342 | if (eval_poly_at(div, xs[i]).val == pieces[i].val) { 343 | corrects++; 344 | } 345 | } 346 | if (corrects < master_degree + errors) { 347 | throw new Error("Answer doesn't match (too many errors)!"); 348 | } 349 | return div; 350 | } 351 | me.berlekamp_welch_attempt = berlekamp_welch_attempt; 352 | 353 | // Extends a list of integers in [0 ... 255] (if using Galois arithmetic) by 354 | // adding n redundant error-correction values 355 | function extend(data, n, arithmetic) { 356 | arithmetic = arithmetic || Galois; 357 | function mk(x) { return new arithmetic(x); } 358 | var data2 = data.map(mk); 359 | var data3 = data.slice(); 360 | var xs = []; 361 | var i; 362 | for (i = 0; i < data.length; i++) { 363 | xs.push(new arithmetic(i)); 364 | } 365 | var poly = berlekamp_welch_attempt(data2, xs, data.length - 1); 366 | for (i = 0; i < n; i++) { 367 | data3.push(eval_poly_at(poly, new arithmetic(data.length + i)).val); 368 | } 369 | return data3; 370 | } 371 | me.extend = extend; 372 | 373 | // Repairs a list of integers in [0 ... 255]. Some integers can be 374 | // erroneous, and you can put null (or undefined) in place of an integer if 375 | // you know that a certain value is defective or missing. Uses the 376 | // Berlekamp-Welch algorithm to do error-correction 377 | function repair(data, datasize, arithmetic) { 378 | arithmetic = arithmetic || Galois; 379 | var vs = []; 380 | var xs = []; 381 | var i; 382 | for (var i = 0; i < data.length; i++) { 383 | if (data[i] != null) { 384 | vs.push(new arithmetic(data[i])); 385 | xs.push(new arithmetic(i)); 386 | } 387 | } 388 | var poly = berlekamp_welch_attempt(vs, xs, datasize - 1); 389 | var result = []; 390 | for (i = 0; i < data.length; i++) { 391 | result.push(eval_poly_at(poly, new arithmetic(i)).val); 392 | } 393 | return result; 394 | } 395 | me.repair = repair; 396 | 397 | function transpose(xs) { 398 | var ys = []; 399 | for (var i = 0; i < xs[0].length; i++) { 400 | var y = []; 401 | for (var j = 0; j < xs.length; j++) { 402 | y.push(xs[j][i]); 403 | } 404 | ys.push(y); 405 | } 406 | return ys; 407 | } 408 | 409 | // Extends a list of bytearrays 410 | // eg. extend_chunks([map(ord, 'hello'), map(ord, 'world')], 2) 411 | // n is the number of redundant error-correction chunks to add 412 | function extend_chunks(data, n, arithmetic) { 413 | arithmetic = arithmetic || Galois; 414 | var o = []; 415 | for (var i = 0; i < data[0].length; i++) { 416 | o.push(extend(data.map(function(x) { return x[i]; }), n, arithmetic)); 417 | } 418 | return transpose(o); 419 | } 420 | me.extend_chunks = extend_chunks; 421 | 422 | // Repairs a list of bytearrays. Use null in place of a missing array. 423 | // Individual arrays can contain some missing or erroneous data. 424 | function repair_chunks(data, datasize, arithmetic) { 425 | arithmetic = arithmetic || Galois; 426 | var first_nonzero = 0; 427 | while (data[first_nonzero] == null) { 428 | first_nonzero++; 429 | } 430 | var i; 431 | for (i = 0; i < data.length; i++) { 432 | if (data[i] == null) { 433 | data[i] = new Array(data[first_nonzero].length); 434 | } 435 | } 436 | var o = []; 437 | for (i = 0; i < data[0].length; i++) { 438 | o.push(repair(data.map(function(x) { return x[i]; }), datasize, arithmetic)); 439 | } 440 | return transpose(o); 441 | } 442 | me.repair_chunks = repair_chunks; 443 | 444 | // Extends either a bytearray or a list of bytearrays or a list of lists... 445 | // Used in the cubify method to expand a cube in all dimensions 446 | function deep_extend_chunks(data, n, arithmetic) { 447 | arithmetic = arithmetic || Galois; 448 | if (!(data[0] instanceof Array)) { 449 | return extend(data, n, arithmetic) 450 | } else { 451 | var o = []; 452 | for (var i = 0; i < data[0].length; i++) { 453 | o.push(deep_extend_chunks( 454 | data.map(function(x) { return x[i]; }), n, arithmetic)); 455 | } 456 | return transpose(o); 457 | } 458 | } 459 | me.deep_extend_chunks = deep_extend_chunks; 460 | 461 | function isObject(o) { 462 | return typeof o == 'object' || typeof o == 'function'; 463 | } 464 | if (typeof define == 'function' && typeof define.amd == 'object' && define.amd) { 465 | define(function() { 466 | return me; 467 | }); 468 | } else { 469 | done = 0 470 | try { 471 | if (isObject(module)) { module.exports = me; } 472 | else (isObject(window) ? window : this).Erasure = me; 473 | } 474 | catch(e) { 475 | (isObject(window) ? window : this).Erasure = me; 476 | } 477 | } 478 | }.call(this)); 479 | -------------------------------------------------------------------------------- /erasure_code/share.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethereum/economic-modeling/320caa16e0d253778131c6c933c7d7330a760634/erasure_code/share.pyc -------------------------------------------------------------------------------- /erasure_code/test.py: -------------------------------------------------------------------------------- 1 | import random 2 | import share 3 | 4 | fsz = 200 5 | 6 | f = ''.join([ 7 | random.choice('1234567890qwetyuiopasdfghjklzxcvbnm') for x in range(fsz)]) 8 | 9 | c = share.split_file(f, 5, 4) 10 | 11 | print 'File split successfully.' 12 | print ' ' 13 | print 'Chunks: ' 14 | print ' ' 15 | for chunk in c: 16 | print chunk 17 | print ' ' 18 | 19 | g = ''.join([ 20 | random.choice('1234567890qwetyuiopasdfghjklzxcvbnm') for x in range(fsz)]) 21 | 22 | c2 = share.split_file(g, 5, 4) 23 | 24 | assert share.recombine_file( 25 | [c[0], c2[1], c[2], c[3], c2[4], c[5], c[6], c[7], c[8]]) == f 26 | 27 | print '5 of 9 with 7 legit, 2 errors passed' 28 | 29 | assert share.recombine_file( 30 | [c[0], c[2], c[3], c2[4], c[6], c[7], c[8]]) == f 31 | 32 | print '5 of 9 with 6 legit, 1 error passed' 33 | 34 | assert share.recombine_file( 35 | [c[0], c[3], c[6], c[7], c[8]]) == f 36 | 37 | print '5 of 9 with 5 legit, 0 errors passed' 38 | 39 | chunks3 = share.serialize_cubify(f, 3, 3, 2) 40 | 41 | print ' ' 42 | print 'Chunks: ' 43 | print ' ' 44 | for chunk in chunks3: 45 | print chunk 46 | print ' ' 47 | 48 | for i in range(26): 49 | pos = random.randrange(len(chunks3)) 50 | print 'Removing cell %d' % pos 51 | chunks3.pop(pos) 52 | 53 | assert share.full_heal_set(chunks3) == f 54 | 55 | print ' ' 56 | print 'Cube reconstruction test passed' 57 | print ' ' 58 | 59 | chunks4 = share.serialize_cubify(g, 3, 3, 2) 60 | 61 | for i in range(7): 62 | pos = random.randrange(len(chunks3)) 63 | chunk = chunks3.pop(pos) 64 | print 'Damaging cell %d' % pos 65 | print 'Prior: %s' % chunk 66 | metadata, content = share.deserialize_chunk(chunk) 67 | for j in range(len(content)): 68 | content[j] = random.randrange(256) 69 | chunks3.append(share.serialize_chunk(content, *metadata)) 70 | print 'Post: %s' % chunks3[-1] 71 | 72 | assert share.full_heal_set(chunks4) == g 73 | 74 | print ' ' 75 | print 'Byzantine cube reconstruction test passed' 76 | -------------------------------------------------------------------------------- /erasure_code/utils.h: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_H_ 2 | #define UTILS_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | // turn a pair of iterators into a range 9 | template 10 | class iter_pair { 11 | T a, b; 12 | typedef typename std::remove_reference(nullptr))>::type value_type; 13 | public: 14 | iter_pair(const T& _a, const T& _b) : a(_a), b(_b) { } 15 | iter_pair(T&& _a, T&& _b) : a(std::move(_a)), b(std::move(_b)) { } 16 | T begin() const { return a; } 17 | T end() const { return b; } 18 | 19 | // for random access iterators 20 | value_type& operator[](std::size_t ix) { return *(a + ix); } 21 | const value_type& operator[](std::size_t ix) const { return *(a + ix); } 22 | std::size_t size() const { return b - a; } 23 | }; 24 | 25 | template 26 | iter_pair make_iter_pair(const T& a, const T& b) { 27 | return iter_pair(a, b); 28 | } 29 | template 30 | iter_pair make_iter_pair(T&& a, T&& b) { 31 | return iter_pair(std::move(a), std::move(b)); 32 | } 33 | 34 | #endif 35 | -------------------------------------------------------------------------------- /ghost.py: -------------------------------------------------------------------------------- 1 | # Time between successful PoW solutions 2 | POW_SOLUTION_TIME = 12 3 | # Time for a block to traverse the network 4 | TRANSIT_TIME = 8 5 | # Max uncle depth 6 | UNCLE_DEPTH = 4 7 | # Uncle block reward (normal block reward = 1) 8 | UNCLE_REWARD_COEFF = 32/32. 9 | UNCLE_DEPTH_PENALTY = 4/32. 10 | # Reward for including uncles 11 | NEPHEW_REWARD_COEFF = 1/32. 12 | # Rounds to test 13 | ROUNDS = 1000000 14 | 15 | import random 16 | 17 | all_miners = {} 18 | 19 | 20 | class Miner(): 21 | def __init__(self, p, backward=0): 22 | # Miner hashpower 23 | self.hashpower = p 24 | # Miner mines a few blocks behind the head? 25 | self.backward = backward 26 | self.id = random.randrange(10000000) 27 | # Set up a few genesis blocks (since the algo is grandpa-dependent, 28 | # we need two genesis blocks plus some genesis uncles) 29 | self.blocks = {} 30 | self.children = {} 31 | for i in range(UNCLE_DEPTH + 2): 32 | self.blocks[i] = \ 33 | {"parent": i-1, "uncles": {}, "miner": -1, "height": i, 34 | "score": i, "id": i} 35 | self.children[i-1] = {i: True} 36 | # ID of "latest block" 37 | self.head = UNCLE_DEPTH + 1 38 | 39 | # Hear about a block 40 | def recv(self, block): 41 | # Add the block to the set if it's valid 42 | addme = True 43 | if block["id"] in self.blocks: 44 | addme = False 45 | if block["parent"] not in self.blocks: 46 | addme = False 47 | if addme: 48 | self.blocks[block["id"]] = block 49 | if block["parent"] not in self.children: 50 | self.children[block["parent"]] = {} 51 | if block["id"] not in self.children[block["parent"]]: 52 | self.children[block["parent"]][block["id"]] = block["id"] 53 | if block["score"] > self.blocks[self.head]["score"]: 54 | self.head = block["id"] 55 | 56 | # Mine a block 57 | def mine(self): 58 | HEAD = self.blocks[self.head] 59 | for i in range(self.backward): 60 | HEAD = self.blocks[HEAD["parent"]] 61 | H = HEAD 62 | h = self.blocks[self.blocks[self.head]["parent"]] 63 | # Select the uncles. The valid set of uncles for a block consists 64 | # of the children of the 2nd to N+1th order grandparents minus 65 | # the parent and said grandparents themselves and blocks that were 66 | # uncles of those previous blocks 67 | u = {} 68 | notu = {} 69 | for i in range(UNCLE_DEPTH - self.backward): 70 | for c in self.children.get(h["id"], {}): 71 | u[c] = True 72 | notu[H["id"]] = True 73 | for c in H["uncles"]: 74 | notu[c] = True 75 | H = h 76 | h = self.blocks[h["parent"]] 77 | for i in notu: 78 | if i in u: 79 | del u[i] 80 | block = {"parent": self.head, "uncles": u, "miner": self.id, 81 | "height": HEAD["height"] + 1, "score": HEAD["score"]+1+len(u), 82 | "id": random.randrange(1000000000000)} 83 | self.recv(block) 84 | global all_miners 85 | all_miners[block["id"]] = block 86 | return block 87 | 88 | 89 | # If b1 is the n-th degree grandchild and b2 is the m-th degree grandchild 90 | # of nearest common ancestor C, returns min(m, n) 91 | def cousin_degree(miner, b1, b2): 92 | while miner.blocks[b1]["height"] > miner.blocks[b2]["height"]: 93 | b1 = miner.blocks[b1]["parent"] 94 | while miner.blocks[b2]["height"] > miner.blocks[b1]["height"]: 95 | b2 = miner.blocks[b2]["parent"] 96 | t = 0 97 | while b1 != b2: 98 | b1 = miner.blocks[b1]["parent"] 99 | b2 = miner.blocks[b2]["parent"] 100 | t += 1 101 | return t 102 | 103 | # Set hashpower percentages and strategies 104 | # Strategy = how many blocks behind head you mine 105 | profiles = [ 106 | # (hashpower, strategy, count) 107 | (1, 0, 20), 108 | (1, 1, 4), # cheaters, mine 1/2/4 blocks back to reduce 109 | (1, 2, 3), # chance of being in a two-block fork 110 | (1, 4, 3), 111 | (5, 0, 1), 112 | (10, 0, 1), 113 | (15, 0, 1), 114 | (25, 0, 1), 115 | ] 116 | 117 | total_pct = 0 118 | miners = [] 119 | for p, b, c in profiles: 120 | for i in range(c): 121 | miners.append(Miner(p, b)) 122 | total_pct += p 123 | 124 | miner_dict = {} 125 | for m in miners: 126 | miner_dict[m.id] = m 127 | 128 | listen_queue = [] 129 | 130 | for t in range(ROUNDS): 131 | if t % 5000 == 0: 132 | print t 133 | for m in miners: 134 | R = random.randrange(POW_SOLUTION_TIME * total_pct) 135 | if R < m.hashpower and t < ROUNDS - TRANSIT_TIME * 3: 136 | b = m.mine() 137 | listen_queue.append([t + TRANSIT_TIME, b]) 138 | while len(listen_queue) and listen_queue[0][0] <= t: 139 | t, b = listen_queue.pop(0) 140 | for m in miners: 141 | m.recv(b) 142 | 143 | h = miners[0].blocks[miners[0].head] 144 | profit = {} 145 | total_blocks_in_chain = 0 146 | length_of_chain = 0 147 | ZORO = {} 148 | print "### PRINTING BLOCKCHAIN ###" 149 | 150 | while h["id"] > UNCLE_DEPTH + 2: 151 | # print h["id"], h["miner"], h["height"], h["score"] 152 | # print "Uncles: ", list(h["uncles"]) 153 | total_blocks_in_chain += 1 + len(h["uncles"]) 154 | ZORO[h["id"]] = True 155 | length_of_chain += 1 156 | profit[h["miner"]] = profit.get(h["miner"], 0) + \ 157 | 1 + NEPHEW_REWARD_COEFF * len(h["uncles"]) 158 | for u in h["uncles"]: 159 | ZORO[u] = True 160 | u2 = miners[0].blocks[u] 161 | profit[u2["miner"]] \ 162 | = profit.get(u2["miner"], 0) + UNCLE_REWARD_COEFF - UNCLE_DEPTH_PENALTY * (h["height"] - u2["height"]) 163 | h = miners[0].blocks[h["parent"]] 164 | 165 | print "### PRINTING HEADS ###" 166 | 167 | for m in miners: 168 | print m.head 169 | 170 | 171 | print "### PRINTING PROFITS ###" 172 | 173 | for p in profit: 174 | print miner_dict.get(p, Miner(0)).hashpower, profit.get(p, 0) 175 | 176 | print "### PRINTING RESULTS ###" 177 | 178 | groupings = {} 179 | counts = {} 180 | for p in profit: 181 | m = miner_dict.get(p, None) 182 | if m: 183 | h = str(m.hashpower)+','+str(m.backward) 184 | counts[h] = counts.get(h, 0) + 1 185 | groupings[h] = groupings.get(h, 0) + profit[p] 186 | 187 | for c in counts: 188 | print c, groupings[c] / counts[c] / (groupings['1,0'] / counts['1,0']) 189 | 190 | print " " 191 | print "Total blocks produced: ", len(all_miners) - UNCLE_DEPTH 192 | print "Total blocks in chain: ", total_blocks_in_chain 193 | print "Efficiency: ", \ 194 | total_blocks_in_chain * 1.0 / (len(all_miners) - UNCLE_DEPTH) 195 | print "Average uncles: ", total_blocks_in_chain * 1.0 / length_of_chain - 1 196 | print "Length of chain: ", length_of_chain 197 | print "Block time: ", ROUNDS * 1.0 / length_of_chain 198 | -------------------------------------------------------------------------------- /ghost_timing_strats.py: -------------------------------------------------------------------------------- 1 | import random 2 | import hashlib 3 | import sys 4 | # Clock offset 5 | CLOCKOFFSET = 1 6 | # Block time 7 | BLKTIME = 15 8 | # Skip time 9 | SKIPTIME = 40 10 | # Round run time 11 | ROUND_RUNTIME = 2000000 12 | # Number of rounds 13 | ROUNDS = 2000 14 | # Block reward 15 | BLKREWARD = 1 16 | # Reward for including x ticks' worth of transactions 17 | # Linear by default, but sublinear formulas are 18 | # probably most accurate 19 | get_txreward = lambda ticks: 1 * ticks 20 | # Latency function 21 | latency_sample = lambda L: int((random.expovariate(1) * ((L/1.33)**0.75))**1.33) 22 | # latency_sample = lambda L: random.randrange(L * 2) 23 | # latency = lambda: random.randrange(15) if random.randrange(10) else 47 24 | # Offline rate 25 | OFFLINE_RATE = 0.23 26 | # Additional block reward for a skip block 27 | SKIP_REWARD = 0 28 | # Score contribution for a too-early block (1, 0 or -1) 29 | TOO_EARLY_SCORE = 1 30 | # Using GHOST? 31 | USING_GHOST = 0 32 | # Default latency 33 | DEFAULT_LATENCY = 7.5 34 | 35 | BLANK_STATE = {'transactions': 0} 36 | 37 | class Simulation(): 38 | def __init__(self, validators): 39 | for i, v in enumerate(validators): 40 | v.id = i 41 | v.simulation = self 42 | self.validators = validators 43 | self.time = 0 44 | self.next_id = 0 45 | self.gvcache = {} 46 | 47 | def run(self, rounds): 48 | # Run the simulation 49 | for i in range(rounds): 50 | for m in self.validators: 51 | m.mine() 52 | m.listen() 53 | self.time += 1 54 | if i % (rounds // 100) == 0: 55 | print 'Completed %d rounds out of %d' % (i, rounds) 56 | 57 | def get_validator(self, randao, skips): 58 | key = (randao << 32) + skips 59 | if key not in self.gvcache: 60 | self.gvcache[key] = sha256_as_int(key) % len(self.validators) 61 | return self.gvcache[key] 62 | 63 | 64 | class Block(): 65 | def __init__(self, parent, state, maker, number=0, skips=0): 66 | self.prevhash = parent.hash if parent else 0 67 | self.state = state 68 | self.number = number 69 | self.height = parent.height + 1 if parent else 0 70 | self.hash = random.randrange(10**20) + 10**23 * self.height 71 | self.randao = sha256_as_int((parent.randao if parent else 0) + maker) 72 | self.totskips = (parent.totskips if parent else 0) + skips 73 | 74 | def sha256_as_int(v): 75 | hashbytes = hashlib.sha256(str(v)).digest()[:4] 76 | o = 0 77 | for b in hashbytes: 78 | o = (o << 8) + ord(b) 79 | return o 80 | 81 | GENESIS = Block(None, BLANK_STATE, 0) 82 | 83 | # Insert a key/value pair into a state 84 | # This is abstracted away into a method to make it easier to 85 | # swap the state out with an immutable dict library or whatever 86 | # else to increase efficiency 87 | def update_state(s, k, v): 88 | s2 = {_k: _v for _k, _v in s.items()} 89 | s2[k] = v 90 | return s2 91 | 92 | # Get a key from a state, default zero 93 | def get_state(s, k): 94 | return s.get(k, 0) 95 | 96 | 97 | class Validator(): 98 | def __init__(self, strategy, latency=5): 99 | # The block that the validator considers to be the head 100 | self.main_chain = [GENESIS.hash] 101 | # A map of tick -> blocks that the validator will receive 102 | # during that tick 103 | self.listen_queue = {} 104 | # Blocks that the validator knows about 105 | self.blocks = {GENESIS.hash: GENESIS} 106 | # Parent -> children map 107 | self.children = {} 108 | # Scores (~= total subtree weight) for those blocks 109 | self.scores = {GENESIS.hash: 0} 110 | # When the validator received each block 111 | self.time_received = {GENESIS.hash: 0} 112 | # Received too early? 113 | self.received_too_early = {} 114 | # Blocks with missing parents, mapping parent hash -> list 115 | self.orphans_by_parent = {} 116 | # ... mapping hash -> list 117 | self.orphans = {} 118 | # This validator's clock is off by this number of ticks 119 | self.time_offset = random.randrange(CLOCKOFFSET) - CLOCKOFFSET // 2 120 | # Set the validator's strategy 121 | self.set_strategy(strategy) 122 | # The validator's ID 123 | self.id = None 124 | # Blocks created 125 | self.created = 0 126 | # Number of blocks to backtrack for GHOST 127 | self.backtrack = 40 128 | # The simulation that this validator is in 129 | self.simulation = None 130 | # Maximum number of skips to try 131 | self.max_skips = 22 132 | # Network latency 133 | self.latency = latency 134 | # Parents on top of which we have already created a block 135 | self.used_parents = {} 136 | # Possible blocks to build on top of 137 | self.heads = [GENESIS] 138 | 139 | def set_strategy(self, strategy): 140 | # The number of ticks a validator waits before producing a block 141 | self.non_skip_produce_delay = strategy[0] 142 | # The number of ticks a validator waits before accepting a block 143 | self.non_skip_accept_delay = strategy[1] 144 | # The number of extra ticks a validator waits per skip (ie. 145 | # if you skip two validator slots then wait this number of ticks 146 | # times two) before producing a block 147 | self.with_skip_produce_delay = strategy[2] 148 | # The number of extra ticks a validator waits per skip before 149 | # accpeint a block 150 | self.with_skip_accept_delay = strategy[3] 151 | 152 | 153 | # Get the time from the validator's point of view 154 | def get_time(self): 155 | return max(self.simulation.time + self.time_offset, 0) 156 | 157 | # Add a block to the listen queue at the given time 158 | def add_to_listen_queue(self, time, obj): 159 | if time not in self.listen_queue: 160 | self.listen_queue[time] = [] 161 | self.listen_queue[time].append(obj) 162 | 163 | def earliest_produce_time(self, parent, skips): 164 | return BLKTIME + (self.non_skip_produce_delay if not skips else 0) + \ 165 | parent.height * BLKTIME + (parent.totskips + skips) * SKIPTIME + \ 166 | (self.with_skip_produce_delay if skips else 0) 167 | 168 | def earliest_accept_time(self, parent, skips): 169 | return BLKTIME + (self.non_skip_accept_delay if not skips else 0) + \ 170 | parent.height * BLKTIME + (parent.totskips + skips) * SKIPTIME + \ 171 | (self.with_skip_accept_delay if skips else 0) 172 | 173 | def mine(self): 174 | # Is it time to produce a block? 175 | t = self.get_time() 176 | for head in self.heads: 177 | skips = 0 178 | while self.simulation.get_validator(head.randao, skips) != self.id and skips < self.max_skips: 179 | skips += 1 180 | if skips == self.max_skips: 181 | return 182 | # If it is... 183 | if t >= self.earliest_produce_time(head, skips) and head.hash not in self.used_parents: 184 | # Can't produce a block at this height anymore 185 | self.used_parents[head.hash] = True 186 | # Small chance to be offline 187 | if random.random() < OFFLINE_RATE: 188 | return 189 | # Compute my block reward 190 | br_key = 'blockrewards:'+str(self.id) 191 | tx_key = 'txfees:'+str(self.id) 192 | skip_key = 'skiprewards:'+str(self.id) 193 | # Claim the reward from the transactions since the parent 194 | new_state = update_state(head.state, 'transactions', self.simulation.time) 195 | # Apply the rewards 196 | new_state = update_state(new_state, br_key, get_state(new_state, br_key) + 1) 197 | new_state = update_state(new_state, skip_key, get_state(new_state, skip_key) + skips) 198 | new_state = update_state(new_state, tx_key, get_state(new_state, tx_key) + 199 | get_txreward(self.simulation.time - head.state['transactions'])) 200 | # Create the block 201 | b = Block(head, new_state, self.id, number=head.number + 1 + skips, skips=skips) 202 | self.created += 1 203 | # print '---------> Validator %d makes block with hash %d and parent %d (%d skips) at time %d' % (self.id, b.hash, b.prevhash, skips, self.simulation.time) 204 | # Broadcast it 205 | for validator in self.simulation.validators: 206 | recv_time = self.simulation.time + 1 + latency_sample(self.latency + validator.latency) 207 | # print 'broadcasting, delay %d' % (recv_time - t) 208 | validator.add_to_listen_queue(recv_time, b) 209 | return 210 | 211 | # If a validator realizes that it "should" have a block but doesn't, 212 | # it can use this method to request it from the network 213 | def request_block(self, hash): 214 | for validator in self.simulation.validators: 215 | if hash in validator.blocks: 216 | recv_time = self.simulation.time + 1 + latency_sample(self.latency + validator.latency) 217 | self.add_to_listen_queue(recv_time, validator.blocks[hash]) 218 | 219 | # Process all blocks that it should receive during the current tick 220 | def listen(self): 221 | if self.simulation.time in self.listen_queue: 222 | for blk in self.listen_queue[self.simulation.time]: 223 | self.accept_block(blk) 224 | if self.simulation.time in self.listen_queue: 225 | del self.listen_queue[self.simulation.time] 226 | 227 | def get_score_addition(self, blk): 228 | parent = self.blocks[blk.prevhash] 229 | skips = blk.number - parent.number - 1 230 | return (TOO_EARLY_SCORE if blk.hash in self.received_too_early else 1) # + random.randrange(100) - 50 231 | 232 | def process_lchain_scores(self, blk): 233 | s = self.get_score_addition(blk) 234 | self.scores[blk.hash] = self.scores.get(blk.prevhash, 0) + s 235 | if self.scores[blk.hash] == self.scores[self.heads[0].hash] and blk not in self.heads: 236 | self.heads.append(blk) 237 | elif self.scores[blk.hash] > self.scores[self.heads[0].hash]: 238 | self.heads = [blk] 239 | 240 | def process_ghost_scores(self, blk): 241 | s = self.get_score_addition(blk) 242 | check_block = blk 243 | fork_block = None 244 | for i in range(self.backtrack): 245 | self.scores[check_block.hash] = self.scores.get(check_block.hash, 0) + s 246 | if check_block.height == 0: 247 | break 248 | elif check_block.height >= len(self.main_chain): 249 | fork_block = self.blocks[check_block.prevhash] 250 | elif check_block.hash != self.main_chain[check_block.height]: 251 | fork_block = self.blocks[check_block.prevhash] 252 | check_block = self.blocks[check_block.prevhash] 253 | # if blk.height - fork_block.height - 1 > 0: 254 | # print 'forking back %d blocks' % (blk.height - fork_block.height - 1) 255 | while len(self.main_chain) > fork_block.height + 1: 256 | self.main_chain.pop() 257 | while 1: 258 | best_child = None 259 | best_score = -1 260 | for c in self.children.get(self.main_chain[-1], []): 261 | if self.scores[c] > best_score: 262 | best_score = self.scores[c] 263 | best_child = c 264 | if best_child is None: 265 | break 266 | self.main_chain.append(best_child) 267 | if len(self.main_chain) == 1: 268 | self.heads = [self.blocks[self.main_chain[0]]] 269 | else: 270 | self.head_parent = self.main_chain[-2] 271 | self.heads = [self.blocks[self.main_chain[-2]]] 272 | best_score = 0 273 | for c in self.children.get(self.main_chain[-2], []): 274 | if self.scores[c] > best_score: 275 | best_score = self.scores[c] 276 | self.heads = [self.blocks[c]] 277 | elif self.scores[c] == best_score: 278 | self.heads.append(self.blocks[c]) 279 | 280 | def accept_block(self, blk): 281 | t = self.get_time() 282 | # Parent not found or at least not yet processed 283 | if blk.prevhash not in self.blocks and blk.hash not in self.orphans: 284 | self.request_block(blk.prevhash) 285 | if blk.prevhash not in self.orphans_by_parent: 286 | self.orphans_by_parent[blk.prevhash] = [] 287 | self.orphans_by_parent[blk.prevhash].append(blk.hash) 288 | self.orphans[blk.hash] = blk 289 | # print 'validator %d skipping block %d: parent %d not found' % (self.id, blk.hash, blk.prevhash) 290 | return 291 | # Already processed? 292 | if blk.hash in self.blocks or blk.hash in self.orphans: 293 | # print 'validator %d skipping block %d: already processed' % (self.id, blk.hash) 294 | return 295 | # Too early? Re-append at earliest allowed time 296 | parent = self.blocks[blk.prevhash] 297 | skips = blk.number - parent.number - 1 298 | alotted_recv_time = self.earliest_accept_time(parent, skips) 299 | if t < alotted_recv_time: 300 | self.received_too_early[blk.hash] = alotted_recv_time - t 301 | self.add_to_listen_queue((alotted_recv_time - t) * 2 + self.simulation.time, blk) 302 | # print 'too early, validator %d delaying %d (%d vs %d)' % (self.id, blk.hash, t, alotted_recv_time) 303 | return 304 | # Add the block and compute the score 305 | # print 'Validator %d receives block, hash %d, time %d' % (self.id, blk.hash, self.simulation.time) 306 | self.blocks[blk.hash] = blk 307 | self.time_received[blk.hash] = t 308 | if parent.hash not in self.children: 309 | self.children[parent.hash] = [] 310 | self.children[parent.hash].append(blk.hash) 311 | if blk.hash in self.orphans: 312 | del self.orphans[blk.hash] 313 | # Process the ghost scoring rule 314 | if USING_GHOST: 315 | self.process_ghost_scores(blk) 316 | else: 317 | self.process_lchain_scores(blk) 318 | # print 'post', self.main_chain 319 | # self.scores[blk.hash] = self.scores[blk.prevhash] + get_score_addition(skips) 320 | if self.orphans_by_parent.get(blk.hash, []): 321 | for c in self.orphans_by_parent[blk.hash]: 322 | # print 'including previously rejected child of %d: %d' % (blk.hash, c) 323 | b = self.orphans[c] 324 | del self.orphans[c] 325 | self.accept_block(b) 326 | del self.orphans_by_parent[blk.hash] 327 | 328 | 329 | def simple_test(baseline=[2, 0, 2, 0]): 330 | # Define the strategies of the validators 331 | strategy_groups = [ 332 | # ((nonskip produce, nonskip accept, skip produce, skip accept) delays, network latency, number of nodes) 333 | ((baseline[0], baseline[1], baseline[2], baseline[3]), DEFAULT_LATENCY, 15), 334 | ((baseline[0] - 0, baseline[1], baseline[2] - 80, baseline[3]), DEFAULT_LATENCY, 4), 335 | ((baseline[0] - 0, baseline[1], baseline[2] - 60, baseline[3]), DEFAULT_LATENCY, 4), 336 | ((baseline[0] - 0, baseline[1], baseline[2] - 40, baseline[3]), DEFAULT_LATENCY, 4), 337 | ((baseline[0] - 0, baseline[1], baseline[2] - 20, baseline[3]), DEFAULT_LATENCY, 4), 338 | # ((baseline[0] + 20, baseline[1], baseline[2] + 20, baseline[3]), DEFAULT_LATENCY, 3), 339 | # ((baseline[0] + 40, baseline[1], baseline[2] + 40, baseline[3]), DEFAULT_LATENCY, 3), 340 | ((baseline[0], baseline[1] - 0, baseline[2], baseline[3] - 80), DEFAULT_LATENCY, 3), 341 | ((baseline[0], baseline[1] - 0, baseline[2], baseline[3] - 60), DEFAULT_LATENCY, 3), 342 | ((baseline[0], baseline[1] - 0, baseline[2], baseline[3] - 40), DEFAULT_LATENCY, 3), 343 | ((baseline[0], baseline[1] - 0, baseline[2], baseline[3] - 20), DEFAULT_LATENCY, 3), 344 | ((baseline[0], baseline[1] + 0, baseline[2], baseline[3] + 0), DEFAULT_LATENCY, 3), 345 | ((baseline[0], baseline[1] + 0, baseline[2], baseline[3] - 20), DEFAULT_LATENCY, 3), 346 | ((baseline[0], baseline[1] + 0, baseline[2], baseline[3] + 40), DEFAULT_LATENCY, 3), 347 | # ((baseline[0], baseline[1] + 40, baseline[2], baseline[3] + 40), DEFAULT_LATENCY, 3), 348 | # ((baseline[0], baseline[1] + 80, baseline[2], baseline[3] + 80), DEFAULT_LATENCY, 3), 349 | ] 350 | sgstarts = [0] 351 | 352 | validators = [] 353 | for s, l, c in strategy_groups: 354 | sgstarts.append(sgstarts[-1] + c) 355 | for i in range(c): 356 | validators.append(Validator(s, l)) 357 | 358 | Simulation(validators).run(ROUND_RUNTIME) 359 | 360 | def report(validators): 361 | head = validators[0].heads[0] 362 | 363 | print 'Head block number:', head.number 364 | print 'Head block height:', head.height 365 | # print validators[0].scores 366 | 367 | for i, ((s, l, c), pos) in enumerate(zip(strategy_groups, sgstarts)): 368 | totblks = 0 369 | tottxs = 0 370 | totskips = 0 371 | totcre = 0 372 | for j in range(pos, pos + c): 373 | totblks += head.state.get('blockrewards:'+str(j), 0) 374 | tottxs += head.state.get('txfees:'+str(j), 0) 375 | totskips += head.state.get('skiprewards:'+str(j), 0) 376 | totcre += validators[j].created 377 | score1 = (totblks * 2.0 - totcre) / c 378 | score2 = (totblks * 2.0 - totcre + tottxs / BLKTIME) / c 379 | print 'Strategy group %d: average %d blocks in chain / %d txfees / %d skips / %d blocks created / %d score1 / %d score2' % (i, totblks * 1.0 / c, tottxs * 1.0 / c, totskips * 1.0 / c, totcre * 1.0 / c, score1, score2) 380 | 381 | report(validators) 382 | 383 | def evo_test(initial_s=[1, 40, 27]): 384 | s0 = [20, 40, 40] 385 | s = initial_s 386 | INITIAL_GROUP = 20 387 | DEVIATION_GROUP = 2 388 | LATENCY = 3 389 | for i in range(ROUNDS): 390 | print 's:', s, ', starting round', i 391 | strategy_groups = [ 392 | (s0, LATENCY, 1), 393 | (s, LATENCY, INITIAL_GROUP - 1), 394 | ] 395 | for j in range(len(s)): 396 | t = [x for x in s] 397 | t[j] = int(t[j] * 1.3) 398 | strategy_groups.append((t, LATENCY, DEVIATION_GROUP)) 399 | u = [x for x in s] 400 | u[j] = int(u[j] * 0.7) 401 | strategy_groups.append((u, LATENCY, DEVIATION_GROUP)) 402 | 403 | sgstarts = [0] 404 | 405 | validators = [] 406 | for _s, _l, c in strategy_groups: 407 | sgstarts.append(sgstarts[-1] + c) 408 | for i in range(c): 409 | validators.append(Validator(_s, _l)) 410 | 411 | Simulation(validators).run(ROUND_RUNTIME) 412 | head = validators[0].head 413 | base_instate = sum([head.state.get(j, 0) for j in range(1, INITIAL_GROUP)]) * 1.0 / (INITIAL_GROUP - 1) 414 | base_totcreated = sum([validators[j].created for j in range(1, INITIAL_GROUP)]) * 1.0 / (INITIAL_GROUP - 1) 415 | base_reward = base_instate * 2 - base_totcreated 416 | print 'old s:', s 417 | for j in range(len(s)): 418 | L = INITIAL_GROUP + DEVIATION_GROUP * 2 * j 419 | M = INITIAL_GROUP + DEVIATION_GROUP * (2 * j + 1) 420 | R = INITIAL_GROUP + DEVIATION_GROUP * (2 * j + 2) 421 | up_instate = sum([head.state.get(k, 0) for k in range(L, M)]) * 1.0 / (M - L) 422 | up_totcreated = sum([validators[k].created for k in range(L, M)]) * 1.0 / (M - L) 423 | up_reward = up_instate * 2 - up_totcreated 424 | down_instate = sum([head.state.get(k, 0) for k in range(M, R)]) * 1.0 / (R - M) 425 | down_totcreated = sum([validators[k].created for k in range(M, R)]) * 1.0 / (R - M) 426 | down_reward = down_instate * 2 - down_totcreated 427 | print 'Adjusting variable %d: %.3f %.3f %.3f' % (j, down_reward, base_reward, up_reward) 428 | if up_reward > base_reward > down_reward: 429 | print 'increasing', j, s 430 | s[j] = int(s[j] * min(1 + (up_reward - base_reward) * 2. / base_reward, 1.2) + 0.49) 431 | elif down_reward > base_reward > up_reward: 432 | print 'decreasing', j, s 433 | s[j] = int(s[j] * max(1 - (down_reward - base_reward) * 2. / base_reward, 0.8) + 0.49) 434 | print 'new s:', s 435 | 436 | if len(sys.argv) >= 2 and sys.argv[1] == "evo": 437 | if len(sys.argv) > 4: 438 | evo_test([int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4])]) 439 | else: 440 | evo_test() 441 | elif len(sys.argv) >= 2 and sys.argv[1] == "onetime": 442 | if len(sys.argv) > 4: 443 | simple_test([int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4])]) 444 | else: 445 | simple_test() 446 | else: 447 | print 'Use evo or onetime' 448 | -------------------------------------------------------------------------------- /inclusive_ghost.py: -------------------------------------------------------------------------------- 1 | import random 2 | GENESIS = 0 3 | LATENCY = 10 4 | CLOCKOFFSET = 10 5 | BLKTIME = 4 6 | RUNTIME = 2000 7 | MINERS = 10 8 | 9 | time = [0] 10 | next_id = [0] 11 | 12 | miners = [] 13 | all_blocks = [] 14 | 15 | class Block(): 16 | def __init__(self, num, parents): 17 | self.num = num 18 | self.parents = parents 19 | if self.num not in all_blocks: 20 | all_blocks.append(self.num) 21 | 22 | def get_ancestors(block, out=None): 23 | if out is None: 24 | out = {} 25 | out[block.num] = True 26 | for p in block.parents: 27 | if p.num not in out: 28 | get_ancestors(p, out) 29 | return out 30 | 31 | class Miner(): 32 | def __init__(self): 33 | self.heads = [Block(-1, [])] 34 | self.listen_queue = {} 35 | self.blocks = {-1: self.heads[0]} 36 | self.scores = {} 37 | self.children = {} 38 | self.id = next_id[0] 39 | next_id[0] += 1 40 | self.time_offset = random.randrange(CLOCKOFFSET) - CLOCKOFFSET // 2 41 | 42 | def get_time(self): 43 | return max(time[0] + self.time_offset, 0) 44 | 45 | def mine(self): 46 | t = self.get_time() 47 | if t % len(miners) == self.id and random.random() < 1.0 / BLKTIME: 48 | new_blk = Block(t, self.heads[:2]) 49 | for h in self.heads: 50 | assert new_blk.num > h.num, (new_blk, self.heads) 51 | print 'new block: %d, parents: %r' % (t, map(lambda x: x.num, self.heads[:2])) 52 | for miner in miners: 53 | recv_time = t + 1 + random.randrange(LATENCY) 54 | if recv_time not in miner.listen_queue: 55 | miner.listen_queue[recv_time] = [] 56 | miner.listen_queue[recv_time].append(new_blk) 57 | 58 | def request_block(self, num): 59 | for miner in miners: 60 | if num in miner.blocks: 61 | recv_time = self.get_time() + 1 + random.randrange(LATENCY) 62 | if recv_time not in self.listen_queue: 63 | self.listen_queue[recv_time] = [] 64 | self.listen_queue[recv_time].append(miner.blocks[num]) 65 | 66 | def listen(self): 67 | t = self.get_time() 68 | if t in self.listen_queue: 69 | for blk in self.listen_queue[t]: 70 | have_parents = True 71 | for p in blk.parents: 72 | if p.num not in self.blocks: 73 | self.request_block(p.num) 74 | have_parents = False 75 | if not have_parents: 76 | print 'no parents :(' 77 | continue 78 | self.blocks[blk.num] = blk 79 | for p in blk.parents: 80 | if p.num not in self.children: 81 | self.children[p.num] = set([]) 82 | self.children[p.num].add(blk.num) 83 | print 'getting ancestors for %r (%d)' % (blk , blk.num) 84 | anc = list(get_ancestors(blk).keys()) 85 | print 'ancestors', anc 86 | for num in anc: 87 | self.scores[num] = self.scores.get(num, 0) + 1 88 | if len(self.scores): 89 | head = self.get_head() 90 | print 'head', head 91 | head_ancestors = get_ancestors(self.blocks[head]) 92 | head2_candidates = [(x,y) for x,y in self.blocks.items() if x not in head_ancestors] 93 | self.heads = [self.blocks[head]] 94 | if len(head2_candidates): 95 | head2 = max(head2_candidates)[1] 96 | self.heads.append(head2) 97 | else: 98 | self.heads = [] 99 | del self.listen_queue[t] 100 | 101 | def get_head(self): 102 | h = -1 103 | while h in self.children and self.children[h]: 104 | best_child = None 105 | best_score = -9999 106 | for c in self.children[h]: 107 | if self.scores[c] > best_score: 108 | best_child = c 109 | best_score = self.scores[c] 110 | h = best_child 111 | return h 112 | 113 | def get_chain(self): 114 | h = [self.get_head()] 115 | while h[-1] >= 0: 116 | h.append(self.blocks[h[-1]].parents[0].num) 117 | return h 118 | 119 | def get_totchain(self, head=None, scanned=None): 120 | if head == None: 121 | head = self.get_head() 122 | scanned = {} 123 | h = [] 124 | scanned[head] = True 125 | for p in self.blocks[head].parents: 126 | if p.num not in scanned: 127 | h.extend([x for x in self.get_totchain(p.num, scanned) if x not in h]) 128 | h.append(head) 129 | return h 130 | 131 | for i in range(MINERS): 132 | miners.append(Miner()) 133 | 134 | for i in range(RUNTIME): 135 | for m in miners: 136 | m.mine() 137 | m.listen() 138 | time[0] += 1 139 | 140 | totcs = [] 141 | 142 | for m in miners: 143 | # print 'chain: %r (%d)' % (m.get_chain(), len(m.get_chain())) 144 | totcs.append(m.get_totchain()) 145 | print 'totchain: %r (%d)' % (totcs[-1], len(totcs[-1])) 146 | if len(totcs) >= 2: 147 | if totcs[-1][:len(totcs[-2])] == totcs[-2][:len(totcs[-1])]: 148 | print 'Perfect match' 149 | else: 150 | print 'Match up to %d' % (min([x for x in range(min(len(totcs[-1]), len(totcs[-2]))) if totcs[-1][x] != totcs[-2][x]]) - 1) 151 | 152 | print 'all blocks: %r (%d)' % (sorted(all_blocks), len(all_blocks)) 153 | -------------------------------------------------------------------------------- /mining/arpow_miner.py: -------------------------------------------------------------------------------- 1 | from pyethereum import utils 2 | import random 3 | 4 | 5 | def sha3(x): 6 | return utils.decode_int(utils.zunpad(utils.sha3(x))) 7 | 8 | 9 | class SeedObj(): 10 | def __init__(self, seed): 11 | self.seed = seed 12 | self.a = 3**160 13 | self.c = 7**80 14 | self.n = 2**256 - 4294968273 # secp256k1n, why not 15 | 16 | def rand(self, r): 17 | self.seed = (self.seed * self.a + self.c) % self.n 18 | return self.seed % r 19 | 20 | 21 | def encode_int(x): 22 | o = '' 23 | for _ in range(8): 24 | o = chr(x % 256) + o 25 | x //= 256 26 | return o 27 | 28 | 29 | ops = { 30 | "plus": lambda x, y: (x + y) % 2**64, 31 | "times": lambda x, y: (x * y) % 2**64, 32 | "xor": lambda x, y: x ^ y, 33 | "and": lambda x, y: x & y, 34 | "or": lambda x, y: x | y, 35 | "not": lambda x, y: 2**64-1-x, 36 | "nxor": lambda x, y: (2**64-1-x) ^ y, 37 | "rshift": lambda x, y: x >> (y % 64) 38 | } 39 | 40 | 41 | def gentape(W, H, SEED): 42 | s = SeedObj(SEED) 43 | tape = [] 44 | for i in range(H): 45 | op = ops.keys()[s.rand(len(ops))] 46 | r = s.rand(100) 47 | if r < 20 and i > 20: 48 | x1 = tape[-r]["x1"] 49 | else: 50 | x1 = s.rand(W) 51 | x2 = s.rand(W) 52 | tape.append({"op": op, "x1": x1, "x2": x2}) 53 | return tape 54 | 55 | 56 | def runtape(TAPE, SEED, params): 57 | s = SeedObj(SEED) 58 | # Fill up tape inputs 59 | mem = [0] * params["w"] 60 | for i in range(params["w"]): 61 | mem[i] = s.rand(2**64) 62 | # Direction variable (run forwards or backwards) 63 | dir = 1 64 | # Run the tape on the inputs 65 | for i in range(params["h"] // 100): 66 | for j in (range(100) if dir == 1 else range(99, -1, -1)): 67 | t = TAPE[i * 100 + j] 68 | mem[t["x1"]] = ops[t["op"]](mem[t["x1"]], mem[t["x2"]]) 69 | # 16% of the time, we flip the order of the next 100 ops; 70 | # this adds in conditional branching 71 | if 2 < mem[t["x1"]] % 37 < 9: 72 | dir *= -1 73 | return sha3(''.join(encode_int(x) for x in mem)) 74 | 75 | 76 | def PoWVerify(header, nonce, params): 77 | tape = gentape(params["w"], params["h"], 78 | sha3(header + encode_int(nonce // params["n"]))) 79 | h = runtape(tape, sha3(header + encode_int(nonce)), params) 80 | print h 81 | return h < 2**256 / params["diff"] 82 | 83 | 84 | def PoWRun(header, params): 85 | # actual randomness, so that miners don't overlap 86 | nonce = random.randrange(2**50) * params["n"] 87 | tape = None 88 | while 1: 89 | print nonce 90 | if nonce % params["n"] == 0: 91 | tape = gentape(params["w"], params["h"], 92 | sha3(header + encode_int(nonce // params["n"]))) 93 | h = runtape(tape, sha3(header + encode_int(nonce)), params) 94 | if h < 2**256 / params["diff"]: 95 | return nonce 96 | else: 97 | nonce += 1 98 | 99 | params = { 100 | "w": 100, 101 | "h": 15000, # generally, w*log(w) at least 102 | "diff": 2**24, # initial 103 | "n": 1000 104 | } 105 | -------------------------------------------------------------------------------- /mining/compute_probabilities_of_finality.py: -------------------------------------------------------------------------------- 1 | import math 2 | BLKTIME = 17 3 | X = 0.28 4 | 5 | faclog = [1] 6 | for i in range(5000): 7 | faclog.append(faclog[-1] * len(faclog)) 8 | 9 | def fac(x): 10 | return faclog[x] 11 | 12 | def poisson(expected, actual): 13 | if expected == 0: 14 | return 1 if actual == 0 else 0 15 | return 2.718281828 ** (-expected + actual * math.log(expected) - math.log(fac(actual))) 16 | 17 | def p_we_win(k, x): 18 | return 1 - (x / (1.0 - x)) ** k 19 | 20 | def p_we_win_after(s): 21 | p = 0 22 | for i in range(4000): 23 | p += poisson(s * 1.0 / BLKTIME, i) * p_we_win(i, X) 24 | return p 25 | 26 | for i in range(0, 7200, 12): 27 | print i, p_we_win_after(i) 28 | -------------------------------------------------------------------------------- /mining/finality_probability_sim.py: -------------------------------------------------------------------------------- 1 | import random 2 | BLKTIME = 600 3 | LATENCY = 10 4 | TEST_MAX = 1200 5 | TEST_INTERVAL = 6 6 | 7 | class Block(): 8 | def __init__(self, parent, txstate): 9 | self.parent = parent 10 | self.score = 1 if parent is None else parent.score + 1 11 | if parent is None or parent.txstate is None: 12 | self.txstate = txstate 13 | else: 14 | self.txstate = parent.txstate 15 | 16 | 17 | results = {} 18 | 19 | 20 | for double_spend_delay in range(0, TEST_MAX, TEST_INTERVAL): 21 | results[double_spend_delay] = 0 22 | for _ in range(1000): 23 | a_head = None 24 | b_head = None 25 | recvqueue = {} 26 | for time_elapsed in range(5000): 27 | txstate = 1 if time_elapsed < double_spend_delay else 2 28 | # Miner A mines and sends (has 50% network share) 29 | if random.random() * BLKTIME < 0.5: 30 | a_head = Block(a_head, txstate) 31 | if time_elapsed + LATENCY not in recvqueue: 32 | recvqueue[time_elapsed + LATENCY] = [] 33 | recvqueue[time_elapsed + LATENCY].append(a_head) 34 | # Miner B mines and sends (has 50% network share) 35 | if random.random() * BLKTIME < 0.5: 36 | b_head = Block(b_head, txstate) 37 | if time_elapsed + LATENCY not in recvqueue: 38 | recvqueue[time_elapsed + LATENCY] = [] 39 | recvqueue[time_elapsed + LATENCY].append(b_head) 40 | # Receive blocks 41 | if time_elapsed in recvqueue: 42 | for b in recvqueue[time_elapsed]: 43 | if not a_head or b.score > a_head.score or (b.score == a_head.score and random.random() < 0.5): 44 | a_head = b 45 | if not b_head or b.score > b_head.score or (b.score == b_head.score and random.random() < 0.5): 46 | b_head = b 47 | # Check which transaction "made it" 48 | if a_head and a_head.txstate == 1: 49 | results[double_spend_delay] += 0.001 50 | print (double_spend_delay, results[double_spend_delay]) 51 | 52 | print(results) 53 | -------------------------------------------------------------------------------- /mining/hashimoto.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Requirements: 5 | - I/O bound: cycles spent on I/O ≫ cycles spent in cpu 6 | - no sharding: impossible to implement data locality strategy 7 | - easy verification 8 | 9 | Thoughts: 10 | 11 | Efficient implementations will not switch context (threading) when waiting for data. 12 | But they would leverage all fill buffers and have concurrent memory accesses. 13 | It can be assumed, that code can be written in a way to calculate N (<10) 14 | nonces in parallel (on a single core). 15 | 16 | So, after all maybe memory bandwidth rather than latency is the actual bottleneck. 17 | Can this be solved in a way that aligns with hashing nonces and allows 18 | for a quick verification? Probably not. 19 | 20 | Loop unrolling: 21 | Initially proposed dagger sets offer data locality which allows to scale the algo 22 | on multiple cores/l2chaches. 320MB / 40sets = 8MB (< L2 cache) 23 | A solution is to make accessed mem location depended on the value of the 24 | previous access. 25 | 26 | Partitial Memory: 27 | If a users only keeps e.g. one third of each DAG in memory (i.e. to 28 | have in L3 cache), he still can answer ~0.5**k of accesses by substituting 29 | them through previous node lookups. 30 | This can be mitigated by 31 | a) making each node deterministically depend on the value of at 32 | least one close high memory node. Optionally for quick validation, select 33 | the 2nd dependency for the lower (cached) memory. see produce_dag_k2dr 34 | b) for DAG creation, using a hashing function which needs more cycles 35 | than multiple memory lookups would - even for GPUs/FPGAs/ASICs. 36 | """ 37 | 38 | 39 | import time 40 | 41 | from pyethereum import utils 42 | 43 | 44 | def decode_int(s): 45 | o = 0 46 | for i in range(len(s)): 47 | o = o * 256 + ord(s[i]) 48 | return o 49 | 50 | 51 | def encode_int(x): 52 | o = '' 53 | for _ in range(64): 54 | o = chr(x % 256) + o 55 | x //= 256 56 | return o 57 | 58 | 59 | def sha3(x): 60 | return decode_int(utils.sha3(x)) 61 | 62 | 63 | def cantor_pair(x, y, p): 64 | return ((x+y) * (x+y+1) / 2 + y) % p 65 | 66 | 67 | def get_daggerset(params, seedset): 68 | return [produce_dag(params, i) for i in seedset] 69 | 70 | 71 | def update_daggerset(params, daggerset, seedset, seed): 72 | idx = decode_int(seed) % len(daggerset) 73 | seedset[idx] = seed 74 | daggerset[idx] = produce_dag(params, seed) 75 | 76 | 77 | def produce_dag(params, seed): 78 | k, hk, w, hw, n, p, t = params.k, params.hk, params.w, \ 79 | params.hw, params.dag_size, params.p, params.h_threshold 80 | print 'Producing dag of size %d (%d memory)' % (n, n * params.wordsz) 81 | o = [sha3(seed)] 82 | init = o[0] 83 | picker = 1 84 | for i in range(1, n): 85 | x = 0 86 | picker = (picker * init) % p 87 | curpicker = picker 88 | if i < t: 89 | for j in range(k): # can be flattend if params are known 90 | x ^= o[curpicker % i] 91 | curpicker >>= 10 92 | else: 93 | for j in range(hk): 94 | x ^= o[curpicker % t] 95 | curpicker >>= 10 96 | o.append(pow(x, w if i < t else hw, p)) # use any "hash function" here 97 | return o 98 | 99 | 100 | def quick_calc(params, seed, pos, known=None): 101 | k, hk, w, hw, p, t = params.k, params.hk, params.w, \ 102 | params.hw, params.p, params.h_threshold 103 | init = sha3(seed) % p 104 | if known is None: 105 | known = {} 106 | known[0] = init 107 | 108 | def calc(i): 109 | if i not in known: 110 | curpicker = pow(init, i, p) 111 | x = 0 112 | if i < t: 113 | for j in range(k): 114 | x ^= calc(curpicker % i) 115 | curpicker >>= 10 116 | known[i] = pow(x, w, p) 117 | else: 118 | for j in range(hk): 119 | x ^= calc(curpicker % t) 120 | curpicker >>= 10 121 | known[i] = pow(x, hw, p) 122 | return known[i] 123 | o = calc(pos) 124 | print 'Calculated index %d in %d lookups' % (pos, len(known)) 125 | return o 126 | 127 | 128 | def hashimoto(params, daggerset, header, nonce): 129 | """ 130 | Requirements: 131 | - I/O bound: cycles spent on I/O ≫ cycles spent in cpu 132 | - no sharding: impossible to implement data locality strategy 133 | 134 | # I/O bound: 135 | e.g. lookups = 16 136 | sha3: 12 * 32 ~384 cycles 137 | lookups: 16 * 160 ~2560 cycles # if zero cache 138 | loop: 16 * 3 ~48 cycles 139 | I/O / cpu = 2560/432 = ~ 6/1 140 | 141 | # no sharding 142 | lookups depend on previous lookup results 143 | impossible to route computation/lookups based on the initial sha3 144 | """ 145 | rand = sha3(header + encode_int(nonce)) % params.p 146 | mix = rand 147 | # loop, that can not be unrolled 148 | # dag and dag[pos] depended on previous lookup 149 | for i in range(params.lookups): 150 | v = mix if params.is_serial else rand >> i 151 | dag = daggerset[v % params.num_dags] # modulo 152 | pos = v % params.dag_size # modulo 153 | mix ^= dag[pos] # xor 154 | # print v % params.num_dags, pos, dag[pos] 155 | print header, nonce, mix 156 | return mix 157 | 158 | 159 | def light_hashimoto(params, seedset, header, nonce): 160 | rand = sha3(header + encode_int(nonce)) % params.p 161 | mix = rand 162 | 163 | for i in range(params.lookups): 164 | v = mix if params.is_serial else rand >> i 165 | seed = seedset[v % len(seedset)] 166 | pos = v % params.dag_size 167 | qc = quick_calc(params, seed, pos) 168 | # print v % params.num_dags, pos, qc 169 | mix ^= qc 170 | print 'Calculated %d lookups' % \ 171 | (params.lookups) 172 | print header, nonce, mix 173 | return mix 174 | 175 | 176 | def light_verify(params, seedset, header, nonce): 177 | h = light_hashimoto(params, seedset, header, nonce) 178 | return h <= 256**params.wordsz / params.diff 179 | 180 | 181 | def mine(daggerset, params, header, nonce=0): 182 | orignonce = nonce 183 | origtime = time.time() 184 | while 1: 185 | h = hashimoto(params, daggerset, header, nonce) 186 | if h <= 256**params.wordsz / params.diff: 187 | noncediff = nonce - orignonce 188 | timediff = time.time() - origtime 189 | print 'Found nonce: %d, tested %d nonces in %.2f seconds (%d per sec)' % \ 190 | (nonce, noncediff, timediff, noncediff / timediff) 191 | return nonce 192 | nonce += 1 193 | 194 | 195 | class params(object): 196 | """ 197 | === tuning === 198 | memory: memory requirements ≫ L2/L3/L4 cache sizes 199 | lookups: hashes_per_sec(lookups=0) ≫ hashes_per_sec(lookups_mem_hard) 200 | k: ? 201 | d: higher values enfore memory availability but require more quick_calcs 202 | num_dags: so that a dag can be updated in reasonable time 203 | """ 204 | p = (2 ** 256 - 4294968273)**2 # prime modulus 205 | wordsz = 64 # word size 206 | memory = 10 * 1024**2 # memory usage 207 | num_dags = 2 # number of dags 208 | dag_size = memory/num_dags/wordsz # num 64byte values per dag 209 | lookups = 40 # memory lookups per hash 210 | diff = 2**14 # higher is harder 211 | k = 2 # num dependecies of each dag value 212 | hk = 8 # dependencies for final nodes 213 | d = 8 # max distance of first dependency (1/d=fraction of size) 214 | w = 2 # work factor on node generation 215 | hw = 8 # work factor on final node generation 216 | h_threshold = dag_size*2/5 # cutoff between final and nonfinal nodes 217 | is_serial = False # hashimoto is serial 218 | 219 | 220 | if __name__ == '__main__': 221 | print dict((k, v) for k, v in params.__dict__.items() 222 | if isinstance(v, int)) 223 | 224 | # odds of a partitial storage attack 225 | missing_mem = 0.01 226 | P_partitial_mem_success = (1-missing_mem) ** params.lookups 227 | print 'P success per hash with %d%% mem missing: %d%%' % \ 228 | (missing_mem*100, P_partitial_mem_success*100) 229 | 230 | # which actually only results in a slower mining, 231 | # as more hashes must be tried 232 | slowdown = 1 / P_partitial_mem_success 233 | print 'x%.1f speedup required to offset %d%% missing mem' % \ 234 | (slowdown, missing_mem*100) 235 | 236 | # create set of DAGs 237 | st = time.time() 238 | seedset = [str(i) for i in range(params.num_dags)] 239 | daggerset = get_daggerset(params, seedset) 240 | print 'daggerset with %d dags' % len(daggerset), 'size:', \ 241 | 64*params.dag_size*params.num_dags / 1024**2, 'MB' 242 | print 'creation took %.2fs' % (time.time() - st) 243 | 244 | # update DAG 245 | st = time.time() 246 | update_daggerset(params, daggerset, seedset, seed='qwe') 247 | print 'updating 1 dag took %.2fs' % (time.time() - st) 248 | 249 | # Mine 250 | for i in range(1): 251 | header = 'test%d' % i 252 | print '\nmining', header 253 | nonce = mine(daggerset, params, header) 254 | # verify 255 | st = time.time() 256 | assert light_verify(params, seedset, header, nonce) 257 | print 'verification took %.2fs' % (time.time() - st) 258 | -------------------------------------------------------------------------------- /mining/mining.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "math/big" 7 | "encoding/hex" 8 | "github.com/obscuren/sha3" 9 | "testing" 10 | "strconv" 11 | ) 12 | 13 | //For use in benchmarking 14 | const tree_depth = 5 15 | const tape_width = 32 //int(math.Pow(2,tree_depth)) < would be nice, but go won't recognize as const 16 | const tape_depth = 100 17 | 18 | //Number of operations that are drawn from to form the tape and the tree 19 | const num_ops = 9 20 | //This is the number of times the PoW algorithm is run 21 | const sample_size = 100 22 | 23 | func Bytes2Hex(d []byte) string { 24 | return hex.EncodeToString(d) 25 | } 26 | 27 | func Hex2Bytes(str string) []byte { 28 | h, _ := hex.DecodeString(str) 29 | return h 30 | } 31 | 32 | func plus(z *big.Int, x *big.Int, y *big.Int) *big.Int { 33 | var lim big.Int 34 | lim.Exp(big.NewInt(2),big.NewInt(256), big.NewInt(0)) 35 | z.Add(x,y) 36 | return z.Mod(z,&lim) 37 | 38 | } 39 | 40 | func times(z *big.Int, x *big.Int, y *big.Int) *big.Int { 41 | var lim, x1, y1 big.Int 42 | lim.Exp(big.NewInt(2),big.NewInt(256), big.NewInt(0)) 43 | x1.Set(x) 44 | y1.Set(y) 45 | z.Mul(x,y) 46 | z.Mod(z,&lim) 47 | return z 48 | } 49 | 50 | func mod(z *big.Int, x *big.Int, y *big.Int) *big.Int { 51 | 52 | if (x.Cmp(big.NewInt(0)) == 0 || y.Cmp(big.NewInt(0)) == 0) { 53 | return big.NewInt(0) 54 | } 55 | if x.Cmp(y) == -1 { //if x < y 56 | z.Mod(y,x) 57 | } else if x.Cmp(y) == 1 { //if x > y 58 | z.Mod(x,y) 59 | } 60 | return z 61 | } 62 | 63 | func xor(z *big.Int, x *big.Int, y *big.Int) *big.Int {return z.Xor(x,y)} 64 | 65 | func nxor(z *big.Int, x *big.Int, y *big.Int) *big.Int { 66 | z.Xor(x,y) 67 | return z.Not(z) 68 | } 69 | 70 | func and(z *big.Int, x *big.Int, y *big.Int) *big.Int {return z.And(x,y)} 71 | 72 | func not(z *big.Int, x *big.Int, y *big.Int) *big.Int { 73 | _ = y 74 | return z.Not(x) 75 | } 76 | 77 | func or(z *big.Int, x *big.Int, y *big.Int) *big.Int {return z.Or(x,y)} 78 | 79 | func rshift(z *big.Int, x *big.Int, y *big.Int) *big.Int { 80 | var lim big.Int 81 | lim.Exp(big.NewInt(2),big.NewInt(256), big.NewInt(0)) 82 | z.Rsh(x,7) 83 | return z.Mod(z,&lim) 84 | } 85 | 86 | 87 | func GetOp(i int) func(z *big.Int, x *big.Int, y *big.Int) *big.Int{ 88 | switch i { 89 | case 0: 90 | return plus 91 | case 1: 92 | return xor 93 | case 2: 94 | return not 95 | case 3: 96 | return times 97 | case 4: 98 | return mod 99 | case 5: 100 | return rshift 101 | case 6: 102 | return or 103 | case 7: 104 | return and 105 | case 8: 106 | return nxor 107 | } 108 | return plus 109 | } 110 | 111 | //the tapelink is used to t 112 | type Tapelink struct { 113 | I int64 114 | J int64 115 | op func(z *big.Int, x *big.Int, y *big.Int) *big.Int 116 | } 117 | 118 | func Sha3Bin(data []byte) []byte { 119 | d := sha3.NewKeccak256() 120 | d.Write(data) 121 | return d.Sum(nil) 122 | } 123 | 124 | func BenchmarkSha3Bin(b *testing.B){ 125 | for i:= 0; i < b.N; i++ { 126 | Sha3Bin([]byte(string(i))) 127 | } 128 | } 129 | 130 | //generates a tape of operations that is w*d long 131 | func gen_tape(seed int64, w int, d int) []Tapelink { 132 | var X = big.NewInt(0) 133 | var Y = big.NewInt(0) 134 | Y.Exp(big.NewInt(2),big.NewInt(80), nil) 135 | var M = big.NewInt(0) 136 | var T []Tapelink 137 | for i := 0; i < w*d; i++{ 138 | // add empty link to tape 139 | T = append(T, *new(Tapelink)) 140 | 141 | // generate new entropy as needed 142 | if (int64(X.Cmp(Y)) == -1) {X.SetBytes(Sha3Bin([]byte(strconv.FormatInt(seed + int64(i), 10))))} 143 | 144 | // Pick random index I 145 | T[i].I = M.Mod(X,big.NewInt(int64(w))).Int64() 146 | M = big.NewInt(int64(w)) 147 | X = X.Div(X,M) 148 | 149 | // Pick random index J 150 | mm := big.NewInt(M.Mod(X,big.NewInt(int64(w-1))).Int64() + int64(1) + T[i].I) 151 | T[i].J = M.Mod(mm, big.NewInt(int64(w))).Int64() 152 | M = big.NewInt(int64(w-1)) 153 | X = X.Div(X,M) 154 | 155 | // Pick random operation 156 | T[i].op = GetOp(int(M.Mod(X, big.NewInt(int64(num_ops))).Int64())) 157 | M = big.NewInt(int64(num_ops)) 158 | X = X.Div(X,M) 159 | } 160 | return T 161 | } 162 | 163 | func BenchmarkGen_tape(b *testing.B){ 164 | var X big.Int 165 | var s int64 166 | b.ResetTimer() 167 | for i := 0; i < b.N; i++ { 168 | b.StopTimer() 169 | X.SetBytes(Sha3Bin([]byte(string(i)))) 170 | s = X.Int64() 171 | b.StartTimer() 172 | gen_tape(s, tape_width, tape_depth) 173 | } 174 | 175 | } 176 | 177 | func gen_inputs(seed int64, w int) []big.Int { 178 | var A []big.Int 179 | for i := 0; i < w; i++ { 180 | A = append(A, *new(big.Int)) 181 | if (i % 256 == 0) { 182 | A[i].SetBytes(Sha3Bin([]byte(strconv.FormatInt(seed + int64(i), 10)))) 183 | } else { 184 | A[i].Lsh(&A[i-1], 1) 185 | } 186 | } 187 | return A 188 | } 189 | 190 | func BenchmarkGen_inputs(b *testing.B){ 191 | var X big.Int 192 | var s int64 193 | b.ResetTimer() 194 | for i := 0; i < b.N; i++ { 195 | b.StopTimer() 196 | X.SetBytes(Sha3Bin([]byte(string(i)))) 197 | s = X.Int64() 198 | b.StartTimer() 199 | gen_inputs(s, tape_width) 200 | } 201 | } 202 | 203 | //this changes the inputs as it goes through a tape with d links 204 | func run_tape(tape []Tapelink, inputs []big.Int, d int) { 205 | var X *big.Int 206 | X = big.NewInt(0) 207 | for i := 0; i < d; i++ { 208 | X = tape[i].op(X, &inputs[tape[i].I], &inputs[tape[i].J]) 209 | inputs[tape[i].I].Set(X) 210 | } 211 | } 212 | 213 | func BenchmarkRun_tape(b *testing.B){ 214 | b.ResetTimer() 215 | for i := 0; i < b.N; i++ { 216 | b.StopTimer() 217 | T := gen_tape(int64(i), tape_width, tape_depth) 218 | I := gen_inputs(int64(i), tape_width) 219 | b.StartTimer() 220 | run_tape(T,I,tape_width*tape_depth) 221 | } 222 | } 223 | 224 | 225 | //returns a 2^d - 1 length tape of operations (no I or J's in the Tapelinks) 226 | func gen_tree(seed int64, d int) []Tapelink { 227 | M := big.NewInt(0) // dummy variable 228 | X := big.NewInt(0) // entropy variable 229 | Y := big.NewInt(0) // entropy buffer size 230 | Y.Exp(big.NewInt(2),big.NewInt(80), nil) 231 | var T []Tapelink // the tree will be stored here 232 | 233 | for i := 0; i < int(math.Pow(2, float64(d))) - 1; i++{ 234 | 235 | T = append(T, *new(Tapelink)) 236 | 237 | //giving it more entropy, if X < 2^32 238 | if (X.Cmp(Y) == -1) {X.SetBytes(Sha3Bin([]byte(strconv.FormatInt(seed + int64(i),10))))} 239 | 240 | //filling the tape with random ops 241 | T[i].op = GetOp(int(M.Mod(X, big.NewInt(num_ops)).Int64())) 242 | M = big.NewInt(num_ops) 243 | X = X.Div(X,M) 244 | } 245 | return T 246 | } 247 | 248 | func BenchmarkGen_tree(b *testing.B) { 249 | var X big.Int 250 | var s int64 251 | b.ResetTimer() 252 | for i := 0; i < b.N; i++ { 253 | b.StopTimer() 254 | X.SetBytes(Sha3Bin([]byte(string(i)))) 255 | s = X.Int64() 256 | b.StartTimer() 257 | gen_tree(s, tree_depth) 258 | } 259 | } 260 | 261 | //there should be 2^d inputs and 2^d - 1 links in the tape. not complying is an unhandled exception 262 | func run_tree(inputs []big.Int, tree []Tapelink, d int) *big.Int { 263 | X := big.NewInt(0) 264 | counter := 0 265 | for j := 0; j < d; j++ { 266 | for i := 0; i < int(math.Pow(2,float64(d - j - 1))); i++ { 267 | X = tree[counter].op(X, &inputs[2*i], &inputs[2*i + 1]) 268 | inputs[i].Set(X) 269 | counter += 1 270 | } 271 | } 272 | 273 | return &inputs[0] 274 | } 275 | 276 | func BenchmarkRun_tree(b *testing.B) { 277 | var X big.Int 278 | b.ResetTimer() 279 | for i := 0; i < b.N; i++ { 280 | b.StopTimer() 281 | var inputs []big.Int 282 | for j := 0; j < tape_width; j++ { 283 | X.SetBytes(Sha3Bin([]byte(string(j + i*tape_width)))) 284 | inputs = append(inputs, X) 285 | } 286 | tree := gen_tree(X.Int64(), tree_depth) 287 | b.StartTimer() 288 | run_tree(inputs, tree, tree_depth) 289 | } 290 | } 291 | 292 | func sha_cap(s []string, n int) []byte{ 293 | var feed string 294 | feed = "" 295 | for i := 0; i < n; i++ { 296 | feed += s[i] 297 | } 298 | return Sha3Bin([]byte(feed)) 299 | } 300 | 301 | 302 | func BenchmarkSha_cap(b *testing.B){ 303 | var X big.Int 304 | var s []string 305 | b.ResetTimer() 306 | for i := 0; i < b.N; i++ { 307 | b.StopTimer() 308 | X.SetBytes(Sha3Bin([]byte(string(i)))) 309 | s = append(s, X.String()) 310 | b.StartTimer() 311 | sha_cap(s,1) 312 | } 313 | } 314 | 315 | func main(){ 316 | var seed int64 317 | var sample []big.Int 318 | 319 | 320 | seed = int64(13300331) 321 | 322 | for i := 0; i < sample_size; i++ { 323 | 324 | Tape := gen_tape(seed, tape_width, tape_depth) 325 | Tree := gen_tree(seed, tree_depth) 326 | 327 | seed += int64(i*i) 328 | if i%10 == 0 { 329 | fmt.Printf("i: %d\n",i) 330 | } 331 | I := gen_inputs(seed, tape_width) 332 | run_tape(Tape, I, tape_depth*tape_width) 333 | 334 | 335 | output := *run_tree(I, Tree, tree_depth) 336 | if output.Cmp(big.NewInt(0)) == 0 { 337 | fmt.Printf("We have a zero from the tree, form operation: \n") 338 | fmt.Println(Tree[len(Tree)-1]) 339 | 340 | } 341 | 342 | 343 | var blockhashes []string 344 | blockhashes = append(blockhashes, output.String()) 345 | 346 | sample = append(sample, output) 347 | H := sha_cap(blockhashes, 1) 348 | 349 | output.SetBytes(H) 350 | //sample = append(sample, output) 351 | } 352 | 353 | var collisions int = 0 354 | for i := 0; i < sample_size; i++ { 355 | //fmt.Println(sample[i]) 356 | for j:=0; j < i; j++ { 357 | //if sample[i] == sample[j] { 358 | if sample[i].Cmp(&sample[j]) == 0 { 359 | collisions += 1 360 | //fmt.Printf("collision on i, j: %d, %d", i, j) 361 | //fmt.Println(sample[i]) 362 | } 363 | } 364 | } 365 | fmt.Printf("number of outputs with same value: %d out of %d\n", (1+int(math.Pow(float64(1+8*collisions),0.5)))/2, sample_size) 366 | } 367 | -------------------------------------------------------------------------------- /mining/mining.py: -------------------------------------------------------------------------------- 1 | import pyethereum 2 | u = pyethereum.utils 3 | import time 4 | 5 | #These are the operations that will end up in the tape 6 | ops = [ 7 | lambda x, y: x+y % 2**256, 8 | lambda x, y: x*y % 2**256, 9 | lambda x, y: x % y if y > 0 else x+y, 10 | lambda x, y: x & y, 11 | lambda x, y: x | y, 12 | lambda x, y: x ^ y 13 | ] 14 | 15 | 16 | ''' 17 | the tape will be 'w' wide and 'd' operations deep 18 | it is a list of triples [i, j, op], later used 19 | in the tape's execution: xi = xi op xj 20 | ''' 21 | def gen_tape(seed, w, d): 22 | tape = [] 23 | h = 0 24 | #Getting as much entropy out of a hash as possible 25 | for i in range(d): 26 | if h < 2**32: 27 | h = u.big_endian_to_int(u.sha3(seed+str(i))) 28 | v1 = h % w 29 | h /= w 30 | v2 = h % w 31 | h /= w 32 | op = ops[h % len(ops)] 33 | h /= len(ops) 34 | tape.append([v1, v2, op]) 35 | return tape 36 | 37 | 38 | def lshift(n): 39 | return 2**255 * (n % 2) + (n / 2) 40 | 41 | #Generates the inputs to and evaluates the tape, the mining nonce can be taken to be in the seed 42 | def gen_inputs(seed,w): 43 | #generating the tape's inputs 44 | v = [] 45 | h = 0 46 | for i in range(w): 47 | if i % 1 == 0: 48 | h = u.big_endian_to_int(u.sha3(seed+str(i))) 49 | else: 50 | h = lshift(h) 51 | v.append(h) 52 | return v 53 | 54 | #Evaluate tape on inputs (incorrect dimension of v is an unhandled exception) 55 | def run_tape(v, tape): 56 | for t in tape: 57 | v[t[0]] = t[2](v[t[0]], v[t[1]]) 58 | #Implemented in a blockchain, any additional hashes or timestamp would be added in the sha 59 | return str(v) 60 | 61 | 62 | # This times the various parts of the hashing function - you can make the tape longer to make tape evaluation dominate 63 | #num_iterations is the number of tapes that are used 64 | #num_tape_evals is the number of nonces allowed, per tape 65 | #tape_w is the width of the tape 66 | #tape_d is the depth of the tape 67 | 68 | def test(num_iterations = 10, num_tape_evals = 1000, tape_w = 100, tape_d = 1000): 69 | time_generating_tape = 0. 70 | time_generating_inputs = 0. 71 | time_evaluating_tape = 0. 72 | time_sha_capping = 0. 73 | for i in range(num_iterations): 74 | t = time.time() 75 | tape = gen_tape(str(i), tape_w, tape_d) 76 | time_generating_tape += time.time() - t 77 | 78 | for j in xrange(num_tape_evals): 79 | t = time.time() 80 | v = gen_inputs(str(j), tape_w) 81 | time_generating_inputs += time.time() - t 82 | 83 | t = time.time() 84 | x = run_tape(v,tape) 85 | time_evaluating_tape += time.time() - t 86 | 87 | t = time.time() 88 | h = u.sha3(x) 89 | time_sha_capping += time.time() - t 90 | 91 | total_time = time_generating_tape + time_generating_inputs + time_evaluating_tape + time_sha_capping 92 | print "% of time generating tape:", time_generating_tape/total_time 93 | print "% of time generating inputs:", time_generating_inputs/total_time 94 | print "% of time evaluating tape:", time_evaluating_tape/total_time 95 | print "% of time sha-capping:", time_sha_capping/total_time 96 | -------------------------------------------------------------------------------- /mining/python_sha3.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # coding: utf-8 3 | 4 | # The Keccak sponge function was designed by Guido Bertoni, Joan Daemen, 5 | # Michaël Peeters and Gilles Van Assche. For more information, feedback or 6 | # questions, please refer to their website: http://keccak.noekeon.org/ 7 | # 8 | # Based on the implementation by Renaud Bauvin, 9 | # from http://keccak.noekeon.org/KeccakInPython-3.0.zip 10 | # 11 | # Modified by Moshe Kaplan to be hashlib-compliant 12 | # 13 | # To the extent possible under law, the implementer has waived all copyright 14 | # and related or neighboring rights to the source code in this file. 15 | # http://creativecommons.org/publicdomain/zero/1.0/ 16 | 17 | 18 | import math 19 | 20 | 21 | def sha3_224(data=None): 22 | return Keccak(c=448, r=1152, n=224, data=data) 23 | 24 | 25 | def sha3_256(data=None): 26 | return Keccak(c=512, r=1088, n=256, data=data) 27 | 28 | 29 | def sha3_384(data=None): 30 | return Keccak(c=768, r=832, n=384, data=data) 31 | 32 | 33 | def sha3_512(data=None): 34 | return Keccak(c=1024, r=576, n=512, data=data) 35 | 36 | 37 | class KeccakError(Exception): 38 | """Custom error Class used in the Keccak implementation""" 39 | 40 | def __init__(self, value): 41 | self.value = value 42 | 43 | def __str__(self): 44 | return repr(self.value) 45 | 46 | 47 | class Keccak: 48 | def __init__(self, r, c, n, data=None): 49 | # Initialize the constants used throughout Keccak 50 | # bitrate 51 | self.r = r 52 | # capacity 53 | self.c = c 54 | # output size 55 | self.n = n 56 | 57 | self.b = r + c 58 | # b = 25*w 59 | self.w = self.b // 25 60 | # 2**l = w 61 | self.l = int(math.log(self.w, 2)) 62 | 63 | self.n_r = 12 + 2 * self.l 64 | 65 | self.block_size = r 66 | self.digest_size = n 67 | 68 | # Initialize the state of the sponge 69 | # The state is made up of 25 words, each word being w bits. 70 | self.S = [[0, 0, 0, 0, 0], 71 | [0, 0, 0, 0, 0], 72 | [0, 0, 0, 0, 0], 73 | [0, 0, 0, 0, 0], 74 | [0, 0, 0, 0, 0]] 75 | 76 | # A string of hexchars, where each char represents 4 bits. 77 | self.buffered_data = "" 78 | 79 | # Store the calculated digest. 80 | # We'll only apply padding and recalculate the hash if it's modified. 81 | self.last_digest = None 82 | 83 | if data: 84 | self.update(data) 85 | 86 | # Constants 87 | 88 | ## Round constants 89 | RC = [0x0000000000000001, 90 | 0x0000000000008082, 91 | 0x800000000000808A, 92 | 0x8000000080008000, 93 | 0x000000000000808B, 94 | 0x0000000080000001, 95 | 0x8000000080008081, 96 | 0x8000000000008009, 97 | 0x000000000000008A, 98 | 0x0000000000000088, 99 | 0x0000000080008009, 100 | 0x000000008000000A, 101 | 0x000000008000808B, 102 | 0x800000000000008B, 103 | 0x8000000000008089, 104 | 0x8000000000008003, 105 | 0x8000000000008002, 106 | 0x8000000000000080, 107 | 0x000000000000800A, 108 | 0x800000008000000A, 109 | 0x8000000080008081, 110 | 0x8000000000008080, 111 | 0x0000000080000001, 112 | 0x8000000080008008] 113 | 114 | ## Rotation offsets 115 | r = [[0, 36, 3, 41, 18], 116 | [1, 44, 10, 45, 2], 117 | [62, 6, 43, 15, 61], 118 | [28, 55, 25, 21, 56], 119 | [27, 20, 39, 8, 14]] 120 | 121 | @staticmethod 122 | def Round(A, RCfixed, w): 123 | """Perform one round of computation as defined in the Keccak-f permutation 124 | 125 | A: current state (5x5 matrix) 126 | RCfixed: value of round constant to use (integer) 127 | """ 128 | 129 | #Initialization of temporary variables 130 | B = [[0, 0, 0, 0, 0], 131 | [0, 0, 0, 0, 0], 132 | [0, 0, 0, 0, 0], 133 | [0, 0, 0, 0, 0], 134 | [0, 0, 0, 0, 0]] 135 | C = [0, 0, 0, 0, 0] 136 | D = [0, 0, 0, 0, 0] 137 | 138 | #Theta step 139 | for x in range(5): 140 | C[x] = A[x][0] ^ A[x][1] ^ A[x][2] ^ A[x][3] ^ A[x][4] 141 | 142 | for x in range(5): 143 | D[x] = C[(x - 1) % 5] ^ _rot(C[(x + 1) % 5], 1, w) 144 | 145 | for x in range(5): 146 | for y in range(5): 147 | A[x][y] = A[x][y] ^ D[x] 148 | 149 | #Rho and Pi steps 150 | for x in range(5): 151 | for y in range(5): 152 | B[y][(2 * x + 3 * y) % 5] = _rot(A[x][y], Keccak.r[x][y], w) 153 | 154 | #Chi step 155 | for x in range(5): 156 | for y in range(5): 157 | A[x][y] = B[x][y] ^ ((~B[(x + 1) % 5][y]) & B[(x + 2) % 5][y]) 158 | 159 | #Iota step 160 | A[0][0] = A[0][0] ^ RCfixed 161 | 162 | return A 163 | 164 | @staticmethod 165 | def KeccakF(A, n_r, w): 166 | """Perform Keccak-f function on the state A 167 | 168 | A: 5x5 matrix containing the state, where each entry is a string of hexchars that is 'w' bits long 169 | n_r: number of rounds 170 | w: word size 171 | """ 172 | 173 | for i in xrange(n_r): 174 | A = Keccak.Round(A, Keccak.RC[i] % (1 << w), w) 175 | 176 | return A 177 | 178 | ### Padding rule 179 | # This is a disgusting piece of code. Clean it. 180 | @staticmethod 181 | def pad10star1(M, n): 182 | """Pad M with the pad10*1 padding rule to reach a length multiple of r bits 183 | 184 | M: message pair (length in bits, string of hex characters ('9AFC...') 185 | n: length in bits (must be a multiple of 8) 186 | Example: pad10star1([60, 'BA594E0FB9EBBD30'],8) returns 'BA594E0FB9EBBD93' 187 | """ 188 | 189 | [my_string_length, my_string] = M 190 | 191 | # Check the parameter n 192 | if n % 8 != 0: 193 | raise KeccakError.KeccakError("n must be a multiple of 8") 194 | 195 | # Check the length of the provided string 196 | if len(my_string) % 2 != 0: 197 | #Pad with one '0' to reach correct length (don't know test 198 | #vectors coding) 199 | my_string += '0' 200 | if my_string_length > (len(my_string) // 2 * 8): 201 | raise KeccakError.KeccakError("the string is too short to contain the number of bits announced") 202 | 203 | nr_bytes_filled = my_string_length // 8 204 | nbr_bits_filled = my_string_length % 8 205 | l = my_string_length % n 206 | if ((n - 8) <= l <= (n - 2)): 207 | if (nbr_bits_filled == 0): 208 | my_byte = 0 209 | else: 210 | my_byte = int(my_string[nr_bytes_filled * 2:nr_bytes_filled * 2 + 2], 16) 211 | my_byte = (my_byte >> (8 - nbr_bits_filled)) 212 | my_byte = my_byte + 2 ** (nbr_bits_filled) + 2 ** 7 213 | my_byte = "%02X" % my_byte 214 | my_string = my_string[0:nr_bytes_filled * 2] + my_byte 215 | else: 216 | if (nbr_bits_filled == 0): 217 | my_byte = 0 218 | else: 219 | my_byte = int(my_string[nr_bytes_filled * 2:nr_bytes_filled * 2 + 2], 16) 220 | my_byte = (my_byte >> (8 - nbr_bits_filled)) 221 | my_byte = my_byte + 2 ** (nbr_bits_filled) 222 | my_byte = "%02X" % my_byte 223 | my_string = my_string[0:nr_bytes_filled * 2] + my_byte 224 | while((8 * len(my_string) // 2) % n < (n - 8)): 225 | my_string = my_string + '00' 226 | my_string = my_string + '80' 227 | 228 | return my_string 229 | 230 | def update(self, arg): 231 | # Update the hash object with the string arg. Repeated calls are equivalent to a single call with the concatenation of all the arguments: m.update(a); m.update(b) is equivalent to m.update(a+b). arg is a normal bytestring. 232 | 233 | self.last_digest = None 234 | # Convert the data into a workable format, and add it to the buffer 235 | self.buffered_data += arg.encode('hex') 236 | 237 | # Absorb any blocks we can: 238 | if len(self.buffered_data) * 4 >= self.r: 239 | extra_bits = len(self.buffered_data) * 4 % self.r 240 | 241 | # An exact fit! 242 | if extra_bits == 0: 243 | P = self.buffered_data 244 | self.buffered_data = "" 245 | else: 246 | # Slice it up into the first r*a bits, for some constant a>=1, and the remaining total-r*a bits. 247 | P = self.buffered_data[:-extra_bits // 4] 248 | self.buffered_data = self.buffered_data[-extra_bits // 4:] 249 | 250 | #Absorbing phase 251 | for i in xrange((len(P) * 8 // 2) // self.r): 252 | to_convert = P[i * (2 * self.r // 8):(i + 1) * (2 * self.r // 8)] + '00' * (self.c // 8) 253 | P_i = _convertStrToTable(to_convert, self.w, self.b) 254 | 255 | # First apply the XOR to the state + block 256 | for y in xrange(5): 257 | for x in xrange(5): 258 | self.S[x][y] = self.S[x][y] ^ P_i[x][y] 259 | # Then apply the block permutation, Keccak-F 260 | self.S = Keccak.KeccakF(self.S, self.n_r, self.w) 261 | 262 | def digest(self): 263 | """Return the digest of the strings passed to the update() method so far. 264 | 265 | This is a string of digest_size bytes which may contain non-ASCII 266 | characters, including null bytes.""" 267 | 268 | if self.last_digest: 269 | return self.last_digest 270 | 271 | # UGLY WARNING 272 | # Handle bytestring/hexstring conversions 273 | M = _build_message_pair(self.buffered_data.decode('hex')) 274 | 275 | # First finish the padding and force the final update: 276 | self.buffered_data = Keccak.pad10star1(M, self.r) 277 | self.update('') 278 | # UGLY WARNING over 279 | 280 | assert len(self.buffered_data) == 0, "Why is there data left in the buffer? %s with length %d" % (self.buffered_data, len(self.buffered_data) * 4) 281 | 282 | # Squeezing time! 283 | Z = '' 284 | outputLength = self.n 285 | while outputLength > 0: 286 | string = _convertTableToStr(self.S, self.w) 287 | # Read the first 'r' bits of the state 288 | Z = Z + string[:self.r * 2 // 8] 289 | outputLength -= self.r 290 | if outputLength > 0: 291 | S = KeccakF(S, verbose) 292 | 293 | self.last_digest = Z[:2 * self.n // 8].decode('hex') 294 | return self.last_digest 295 | 296 | def hexdigest(self): 297 | """Like digest() except the digest is returned as a string of hex digits 298 | 299 | This may be used to exchange the value safely in email or other 300 | non-binary environments.""" 301 | return self.digest().encode('hex') 302 | 303 | def copy(self): 304 | # First initialize whatever can be done normally 305 | duplicate = Keccak(c=self.c, r=self.r, n=self.n) 306 | # Then copy over the state. 307 | for i in xrange(5): 308 | for j in xrange(5): 309 | duplicate.S[i][j] = self.S[i][j] 310 | # and any other stored data 311 | duplicate.buffered_data = self.buffered_data 312 | duplicate.last_digest = self.last_digest 313 | return duplicate 314 | 315 | 316 | ## Generic utility functions 317 | 318 | def _build_message_pair(data): 319 | hex_data = data.encode('hex') 320 | size = len(hex_data) * 4 321 | return (size, hex_data) 322 | 323 | 324 | def _rot(x, shift_amount, length): 325 | """Rotate x shift_amount bits to the left, considering the \ 326 | string of bits is length bits long""" 327 | 328 | shift_amount = shift_amount % length 329 | return ((x >> (length - shift_amount)) + (x << shift_amount)) % (1 << length) 330 | 331 | ### Conversion functions String <-> Table (and vice-versa) 332 | 333 | 334 | def _fromHexStringToLane(string): 335 | """Convert a string of bytes written in hexadecimal to a lane value""" 336 | 337 | #Check that the string has an even number of characters i.e. whole number of bytes 338 | if len(string) % 2 != 0: 339 | raise KeccakError.KeccakError("The provided string does not end with a full byte") 340 | 341 | #Perform the conversion 342 | temp = '' 343 | nrBytes = len(string) // 2 344 | for i in xrange(nrBytes): 345 | offset = (nrBytes - i - 1) * 2 346 | temp += string[offset:offset + 2] 347 | return int(temp, 16) 348 | 349 | 350 | def _fromLaneToHexString(lane, w): 351 | """Convert a lane value to a string of bytes written in hexadecimal""" 352 | 353 | laneHexBE = (("%%0%dX" % (w // 4)) % lane) 354 | #Perform the conversion 355 | temp = '' 356 | nrBytes = len(laneHexBE) // 2 357 | for i in xrange(nrBytes): 358 | offset = (nrBytes - i - 1) * 2 359 | temp += laneHexBE[offset:offset + 2] 360 | return temp.upper() 361 | 362 | 363 | def _convertStrToTable(string, w, b): 364 | """Convert a string of hex-chars to its 5x5 matrix representation 365 | 366 | string: string of bytes of hex-coded bytes (e.g. '9A2C...')""" 367 | 368 | # Check that the input paramaters are expected 369 | if w % 8 != 0: 370 | raise KeccakError("w is not a multiple of 8") 371 | 372 | # Each character in the string represents 4 bits. 373 | # The string should have exactly 'b' bits. 374 | if len(string) * 4 != b: 375 | raise KeccakError.KeccakError("string can't be divided in 25 blocks of w bits\ 376 | i.e. string must have exactly b bits") 377 | 378 | #Convert 379 | output = [[0, 0, 0, 0, 0], 380 | [0, 0, 0, 0, 0], 381 | [0, 0, 0, 0, 0], 382 | [0, 0, 0, 0, 0], 383 | [0, 0, 0, 0, 0]] 384 | 385 | bits_per_char = 2 * w // 8 386 | for x in xrange(5): 387 | for y in xrange(5): 388 | # Each entry will have b/25=w bits. 389 | offset = (5 * y + x) * bits_per_char 390 | # Store the data into the associated word. 391 | hexstring = string[offset:offset + bits_per_char] 392 | output[x][y] = _fromHexStringToLane(hexstring) 393 | return output 394 | 395 | 396 | def _convertTableToStr(table, w): 397 | """Convert a 5x5 matrix representation to its string representation""" 398 | 399 | #Check input format 400 | if w % 8 != 0: 401 | raise KeccakError.KeccakError("w is not a multiple of 8") 402 | if (len(table) != 5) or any(len(row) != 5 for row in table): 403 | raise KeccakError.KeccakError("table must be 5x5") 404 | 405 | #Convert 406 | output = [''] * 25 407 | for x in xrange(5): 408 | for y in xrange(5): 409 | output[5 * y + x] = _fromLaneToHexString(table[x][y], w) 410 | output = ''.join(output).upper() 411 | return output 412 | -------------------------------------------------------------------------------- /multi_uncle_ghost.py: -------------------------------------------------------------------------------- 1 | # Time between successful PoW solutions 2 | POW_SOLUTION_TIME = 10 3 | # Time for a block to traverse the network 4 | TRANSIT_TIME = 50 5 | # Number of required uncles 6 | UNCLES = 4 7 | # Uncle block reward (normal block reward = 1) 8 | UNCLE_REWARD_COEFF = 0.875 9 | # Reward for including uncles 10 | NEPHEW_REWARD_COEFF = 0.01 11 | # Rounds to test 12 | ROUNDS = 80000 13 | 14 | import random 15 | import copy 16 | 17 | 18 | class Miner(): 19 | def __init__(self, p): 20 | self.hashpower = p 21 | self.id = random.randrange(10000000) 22 | # Set up a few genesis blocks (since the algo is grandpa-dependent, 23 | # we need two genesis blocks plus some genesis uncles) 24 | self.blocks = { 25 | 0: {"parent": -1, "uncles": [], "miner": -1, "height": 0, 26 | "score": 0, "id": 0, "children": {1: 1}}, 27 | 1: {"parent": 0, "uncles": [], "miner": -1, "height": 1, 28 | "score": 0, "id": 1, "children": {}} 29 | } 30 | # ID of "latest block" 31 | self.head = 1 32 | 33 | # Hear about a block 34 | def recv(self, block): 35 | # Add the block to the set if it's valid 36 | addme = True 37 | if block["id"] in self.blocks: 38 | addme = False 39 | if block["parent"] not in self.blocks: 40 | addme = False 41 | for u in block["uncles"]: 42 | if u not in self.blocks: 43 | addme = False 44 | p = self.blocks[block["parent"]] 45 | if addme: 46 | self.blocks[block["id"]] = copy.deepcopy(block) 47 | # Each parent keeps track of its children, to help 48 | # facilitate the rule that a block must have N+ siblings 49 | # to be valid 50 | if block["id"] not in p["children"]: 51 | p["children"][block["id"]] = block["id"] 52 | # Check if the new block deserves to be the new head 53 | if len(p["children"]) >= 1 + UNCLES: 54 | for c in p["children"]: 55 | newblock = self.blocks[c] 56 | if newblock["score"] > self.blocks[self.head]["score"]: 57 | self.head = newblock["id"] 58 | 59 | # Mine a block 60 | def mine(self): 61 | h = self.blocks[self.blocks[self.head]["parent"]] 62 | b = sorted(list(h["children"]), key=lambda x: -self.blocks[x]["score"]) 63 | p = self.blocks[b[0]] 64 | block = {"parent": b[0], "uncles": b[1:], "miner": self.id, 65 | "height": h["height"] + 2, "score": p["score"] + len(b), 66 | "id": random.randrange(1000000000000), "children": {}} 67 | self.recv(block) 68 | return block 69 | 70 | 71 | def cousin_degree(miner, b1, b2): 72 | while miner.blocks[b1]["height"] > miner.blocks[b2]["height"]: 73 | b1 = miner.blocks[b1]["parent"] 74 | while miner.blocks[b2]["height"] > miner.blocks[b1]["height"]: 75 | b2 = miner.blocks[b2]["parent"] 76 | t = 0 77 | while b1 != b2: 78 | b1 = miner.blocks[b1]["parent"] 79 | b2 = miner.blocks[b2]["parent"] 80 | t += 1 81 | return t 82 | 83 | percentages = [1]*25 + [5, 5, 5, 5, 5, 10, 15, 25] 84 | miners = [] 85 | for p in percentages: 86 | miners.append(Miner(p)) 87 | 88 | miner_dict = {} 89 | for m in miners: 90 | miner_dict[m.id] = m 91 | 92 | listen_queue = [] 93 | 94 | for t in range(ROUNDS): 95 | if t % 5000 == 0: 96 | print t 97 | for m in miners: 98 | R = random.randrange(POW_SOLUTION_TIME * sum(percentages)) 99 | if R < m.hashpower and t < ROUNDS - TRANSIT_TIME * 3: 100 | b = m.mine() 101 | listen_queue.append([t + TRANSIT_TIME, b]) 102 | while len(listen_queue) and listen_queue[0][0] <= t: 103 | t, b = listen_queue.pop(0) 104 | for m in miners: 105 | m.recv(b) 106 | 107 | h = miners[0].blocks[miners[0].head] 108 | profit = {} 109 | total_blocks_in_chain = 0 110 | length_of_chain = 0 111 | ZORO = {} 112 | print "### PRINTING BLOCKCHAIN ###" 113 | 114 | while h["id"] > 1: 115 | print h["miner"], h["height"], h["score"] 116 | total_blocks_in_chain += 1 + len(h["uncles"]) 117 | ZORO[h["id"]] = True 118 | length_of_chain += 1 119 | profit[h["miner"]] = profit.get(h["miner"], 0) + \ 120 | 1 + NEPHEW_REWARD_COEFF * len(h["uncles"]) 121 | for u in h["uncles"]: 122 | ZORO[u] = True 123 | u2 = miners[0].blocks[u] 124 | profit[u2["miner"]] = profit.get(u2["miner"], 0) + UNCLE_REWARD_COEFF 125 | h = miners[0].blocks[h["parent"]] 126 | 127 | print "### PRINTING HEADS ###" 128 | 129 | for m in miners: 130 | print m.head 131 | 132 | 133 | print "### PRINTING PROFITS ###" 134 | 135 | for p in profit: 136 | print miner_dict[p].hashpower, profit[p] 137 | 138 | print "### PRINTING RESULTS ###" 139 | 140 | groupings = {} 141 | counts = {} 142 | for p in profit: 143 | h = miner_dict[p].hashpower 144 | counts[h] = counts.get(h, 0) + 1 145 | groupings[h] = groupings.get(h, 0) + profit[p] 146 | 147 | for c in counts: 148 | print c, groupings[c] / counts[c] / (groupings[1] / counts[1]) 149 | 150 | print " " 151 | print "Total blocks produced: ", len(miners[0].blocks) - 2 152 | print "Total blocks in chain: ", total_blocks_in_chain 153 | print "Efficiency: ", total_blocks_in_chain * 1.0 / (len(miners[0].blocks) - 2) 154 | print "Average uncles: ", total_blocks_in_chain * 1.0 / length_of_chain 155 | print "Length of chain: ", length_of_chain 156 | print "Block time: ", ROUNDS * 1.0 / length_of_chain 157 | -------------------------------------------------------------------------------- /selfish_mining_strats.py: -------------------------------------------------------------------------------- 1 | import random 2 | import sys 3 | 4 | def test_strat(strat, hashpower, gamma, reward, fees, uncle_rewards=1, uncle_coeff=0, rounds=25000): 5 | # Block reward for attacker 6 | me_reward = 0 7 | # Block reward for others 8 | them_reward = 0 9 | # Fees for the attacker 10 | me_fees = 0 11 | # Fees for others 12 | them_fees = 0 13 | # Blocks in current private chain 14 | me_blocks = 0 15 | # Blocks in current public chain 16 | them_blocks = 0 17 | # Time elapsed since last chain merging 18 | time_elapsed = 0 19 | # Divisor for block rewards (diff adjustment) 20 | divisor = 0 21 | # Simulate the system 22 | for i in range(rounds): 23 | # Attacker makes a block 24 | if random.random() < hashpower: 25 | me_blocks += 1 26 | last_is_me = 1 27 | # Honest nodes make a block 28 | else: 29 | them_blocks += 1 30 | last_is_me = 0 31 | time_elapsed += random.expovariate(1) 32 | # "Adopt" or "override" 33 | if me_blocks >= len(strat) or them_blocks >= len(strat[me_blocks]) or strat[me_blocks][them_blocks] == 1: 34 | # Override 35 | if me_blocks > them_blocks or (me_blocks == them_blocks and random.random() < gamma): 36 | me_reward += me_blocks * reward 37 | me_fees += time_elapsed * fees 38 | divisor += me_blocks 39 | # Add uncles 40 | while me_blocks < 7 and them_blocks > 0: 41 | r = min(them_blocks, 2) * (0.875 - 0.125 * me_blocks) * uncle_rewards 42 | divisor += min(them_blocks, 2) * uncle_coeff 43 | them_reward = them_reward + r 44 | them_blocks, me_blocks = them_blocks - 2, me_blocks + 1 45 | # Adopt 46 | else: 47 | them_reward += them_blocks * reward 48 | them_fees += time_elapsed * fees 49 | divisor += them_blocks 50 | # Add uncles 51 | while them_blocks < 7 and me_blocks > 0: 52 | r = min(me_blocks, 2) * (0.875 - 0.125 * them_blocks) * uncle_rewards 53 | divisor += min(me_blocks, 2) * uncle_coeff 54 | me_reward = me_reward + r 55 | me_blocks, them_blocks = me_blocks - 2, them_blocks + 1 56 | me_blocks = 0 57 | them_blocks = 0 58 | time_elapsed = 0 59 | # Match 60 | elif strat[me_blocks][them_blocks] == 2 and not last_is_me: 61 | if random.random() < gamma: 62 | me_reward += me_blocks * reward + time_elapsed * fees 63 | divisor += me_blocks 64 | me_blocks = 0 65 | them_blocks = 0 66 | time_elapsed = 0 67 | # Add uncles 68 | while me_blocks < 7 and them_blocks > 0: 69 | r = min(them_blocks, 2) * (0.875 - 0.125 * me_blocks) 70 | divisor += min(them_blocks, 2) * uncle_coeff 71 | them_reward = them_reward + r 72 | them_blocks, me_blocks = them_blocks - 2, me_blocks + 1 73 | return me_reward / divisor + me_fees / rounds, them_reward / divisor + them_fees / rounds 74 | 75 | # A 20x20 array meaning "what to do if I made i blocks and the network 76 | # made j blocks?". 1 = publish, 0 = do nothing. 77 | def gen_selfish_mining_strat(): 78 | o = [([0] * 20) for i in range(20)] 79 | for me in range(20): 80 | for them in range(20): 81 | # Adopt 82 | if them == 1 and me == 0: 83 | o[me][them] = 1 84 | if them == me + 1: 85 | o[me][them] = 1 86 | # Overtake 87 | if me >= 2 and me == them + 1: 88 | o[me][them] = 1 89 | # Match 90 | if me >= 1 and me == them: 91 | o[me][them] = 2 92 | return o 93 | 94 | 95 | dic = {"rewards": 1, "fees": 0, "gamma": 0.5, "uncle_coeff": 0, "uncle_rewards": 0} 96 | for a in sys.argv[1:]: 97 | param, val = a[:a.index('=')], a[a.index('=')+1:] 98 | dic[param] = float(val) 99 | print dic 100 | s = gen_selfish_mining_strat() 101 | for i in range(1, 50): 102 | x, y = test_strat(s, i * 0.01, dic["gamma"], dic["rewards"], dic["fees"], dic["uncle_rewards"], dic["uncle_coeff"], rounds=200000) 103 | print '%d%% hashpower, %f%% of rewards, (%f attacker, %f honest)' % \ 104 | (i, x * 100.0 / (x + y), x * 100.0 / i, y * 100.0 / (100-i)) 105 | -------------------------------------------------------------------------------- /slasher_v2_sim.py: -------------------------------------------------------------------------------- 1 | NUMSIGNERS = 15 2 | ATTACKER_SHARE = 0.495 3 | CHANCE_OF_SUCCESS = 0.049 4 | SCORE_DIFFERENTIAL = 10 5 | ATTACKER_VOTE = 0.95 6 | import random 7 | 8 | def sim(): 9 | d = -SCORE_DIFFERENTIAL * 15 10 | while d < 0 and d > -(SCORE_DIFFERENTIAL * 15)-1000: 11 | if random.random() < ATTACKER_SHARE: 12 | for i in range(NUMSIGNERS): 13 | if random.random() < ATTACKER_SHARE: 14 | d += ATTACKER_VOTE 15 | else: 16 | d += min(CHANCE_OF_SUCCESS, 0.95) 17 | else: 18 | for i in range(NUMSIGNERS): 19 | if random.random() < ATTACKER_SHARE: 20 | pass 21 | else: 22 | d -= min(1 - CHANCE_OF_SUCCESS, 0.95) 23 | return 1 if d >= 0 else 0 24 | -------------------------------------------------------------------------------- /slasher_withholding_exploit.py: -------------------------------------------------------------------------------- 1 | def fac(n): return 1 if n==0 else n * fac(n-1) 2 | def choose(n,k): return fac(n) / fac(k) / fac(n-k) 3 | def prob(n,k,p): return choose(n,k) * p ** k * (1-p) ** (n-k) 4 | 5 | def prob_lt(n,k,p): return sum([prob(n,i,p) for i in range(p)]) 6 | 7 | SIGS = 30 8 | ACTUALSIGS = 10 9 | POWRETURN = 0.03 10 | POSRETURN = 0.01 11 | 12 | # Expected number of signatures on a block 13 | def ev(pos): 14 | return SIGS * pos 15 | 16 | # Chance you have at least k sigs 17 | def at_least_k(pos, k): 18 | return sum([prob(SIGS, i, pos) for i in range(k, SIGS + 1)]) 19 | 20 | # Expected number of signatures on a block filtering all fac: 24 | o.append(o[-1] * diffs[i] * 1.0 / diffs[i-1] / fac) 25 | elif diffs[i] > diffs[i-1]: 26 | o.append(o[-1]) 27 | else: 28 | o.append(o[-1] * diffs[i] * 1.0 / diffs[i-1]) 29 | return o 30 | 31 | 32 | def diff_estimator(fac, dw, mf, exp=1): 33 | o = [1] 34 | derivs = [0] * 14 35 | for i in range(14, len(diffs)): 36 | derivs.append(diffs[i] - diffs[i - 14]) 37 | for i in range(0, 14): 38 | derivs[i] = derivs[14] 39 | vals = [max(diffs[i] + derivs[i] * dw, diffs[i] * mf) for i in range(len(diffs))] 40 | for i in range(1, len(diffs)): 41 | if vals[i] * 1.0 / vals[i-1] > fac: 42 | o.append(o[-1] * 1.0 / fac * (vals[i] / vals[i-1])**exp) 43 | elif vals[i] > vals[i-1]: 44 | o.append(o[-1]) 45 | else: 46 | o.append(o[-1] * 1.0 * (vals[i] / vals[i-1])**exp) 47 | return o 48 | 49 | 50 | def tx_diff_estimator(fac, dw, mf, lin=1, exp=1): 51 | fac = (fac - 1) or 0.000001 52 | o = [1] 53 | initavg = sum([txs[i] for i in range(5)]) / 5.0 54 | txavgs = [initavg] * 5 55 | for i in range(5, len(txs)): 56 | txavgs.append(txavgs[-1] * 0.8 + txs[i] * 0.2) 57 | 58 | derivs = [0] * 14 59 | for i in range(14, len(txavgs)): 60 | derivs.append(txavgs[i] - txavgs[i - 14]) 61 | for i in range(0, 14): 62 | derivs[i] = derivs[14] 63 | vals = [max(txavgs[i] + derivs[i] * dw, txavgs[i] * mf) for i in range(len(txavgs))] 64 | for i in range(1, len(txavgs)): 65 | growth = (vals[i] * 1.0 / vals[i-1] - 1) 66 | if growth > fac: 67 | surplus = (growth / fac) - 1 68 | o.append(o[-1] * (1 + (surplus * lin * fac) ** exp)) 69 | elif vals[i] > vals[i-1]: 70 | o.append(o[-1]) 71 | else: 72 | surplus = 1 - growth 73 | o.append(o[-1] * (1 - (surplus * lin * fac) ** exp)) 74 | if i and o[-1] < o[-2] * mf: 75 | o[-1] = o[-2] * mf 76 | return o 77 | 78 | 79 | def minimax_fee_estimator(fac, days): 80 | o = [1] 81 | initavg = sum([txs[i] for i in range(int(days))]) * 1.0 / days 82 | txavgs = [initavg] * int(days) 83 | for i in range(int(days), len(txs)): 84 | txavgs.append(txavgs[-1] * 1.0 * (days-1) / days + txs[i] * 1.0 / days) 85 | initavg2 = sum([txfees[i] for i in range(int(days))]) * 1.0 / days 86 | txfeeavgs = [initavg2] * int(days) 87 | for i in range(int(days), len(txs)): 88 | txfeeavgs.append(txfeeavgs[-1] * 1.0 * (days-1) / days + txfees[i] * 1.0 / days) 89 | # Calculate inverse fee, invfee ~= price 90 | txavgfees = [t / f for f, t in zip(txfeeavgs, txavgs)] 91 | for i in range(1, len(txavgfees)): 92 | if txavgfees[i] * 1.0 / txavgfees[i-1] > fac: 93 | o.append(o[-1] * txavgfees[i] * 1.0 / txavgfees[i-1] / fac) 94 | elif txavgfees[i] > txavgfees[i-1]: 95 | o.append(o[-1]) 96 | else: 97 | o.append(o[-1] * txavgfees[i] * 1.0 / txavgfees[i-1]) 98 | return o 99 | 100 | 101 | def ndiff_estimator(*args): 102 | fac, dws, mf = args[0], args[1:-1], args[-1] 103 | o = [1] 104 | ds = [diffs] 105 | for dw in dws: 106 | derivs = [0] * 14 107 | for i in range(14, len(diffs)): 108 | derivs.append(ds[-1][i] - ds[-1][i - 14]) 109 | for i in range(0, 14): 110 | derivs[i] = derivs[14] 111 | ds.append(derivs) 112 | vals = [] 113 | for i in range(len(diffs)): 114 | q = ds[0][i] + sum([ds[j+1][i] * dws[j] for j in range(len(dws))]) 115 | vals.append(max(q, ds[0][i] * mf)) 116 | for i in range(1, len(diffs)): 117 | if vals[i] * 1.0 / vals[i-1] > fac: 118 | o.append(o[-1] * vals[i] * 1.0 / vals[i-1] / fac) 119 | elif vals[i] > vals[i-1]: 120 | o.append(o[-1]) 121 | else: 122 | o.append(o[-1] * vals[i] * 1.0 / vals[i-1]) 123 | return o 124 | 125 | 126 | def dual_threshold_estimator(fac1, fac2, dmul): 127 | o = [1] 128 | derivs = [0] * 14 129 | for i in range(14, len(diffs)): 130 | derivs.append(diffs[i] - diffs[i - 14]) 131 | for i in range(0, 14): 132 | derivs[i] = derivs[14] 133 | for i in range(1, len(diffs)): 134 | if diffs[i] * 1.0 / diffs[i-1] > fac1 and derivs[i] * 1.0 / derivs[i-1] > fac2: 135 | o.append(o[-1] * diffs[i] * 1.0 / diffs[i-1] / fac1 * (1 + (derivs[i] / derivs[i-1] - fac2) * dmul)) 136 | elif diffs[i] > diffs[i-1]: 137 | o.append(o[-1]) 138 | else: 139 | o.append(o[-1] * diffs[i] * 1.0 / diffs[i-1]) 140 | return o 141 | 142 | infinity = 2.**1023 143 | infinity *= 2 144 | 145 | 146 | def evaluate_estimates(estimates, crossvalidate=False): 147 | sz = len(prices) if crossvalidate else 780 148 | sqdiffsum = 0 149 | # compute average 150 | tot = 0 151 | for i in range(sz): 152 | if estimates[i] == infinity or estimates[i] <= 0: 153 | return 10**20 154 | tot += math.log(prices[i] / estimates[i]) 155 | avg = 2.718281828459 ** (tot * 1.0 / sz) 156 | if avg <= 0: 157 | return 10**20 158 | for i in range(1, sz): 159 | sqdiffsum += math.log(prices[i] / estimates[i] / avg) ** 2 160 | return sqdiffsum 161 | 162 | 163 | # Simulated annealing optimizer 164 | def optimize(producer, floors, ceilings, rate=0.7, rounds=5000, tries=1): 165 | bestvals, besty = None, 10**21 166 | for t in range(tries): 167 | print 'Starting test %d of %d' % (t + 1, tries) 168 | vals = [f*0.5+c*0.5 for f, c in zip(floors, ceilings)] 169 | y = evaluate_estimates(producer(*vals)) 170 | for i in range(1, rounds): 171 | stepsizes = [(f*0.5-c*0.5) / i**rate for f, c in zip(floors, ceilings)] 172 | steps = [(random.random() * 2 - 1) * s for s in stepsizes] 173 | newvals = [max(mi, min(ma, v+s)) for v, s, mi, ma in zip(vals, steps, floors, ceilings)] 174 | newy = evaluate_estimates(producer(*newvals)) 175 | if newy < y: 176 | vals = newvals 177 | y = newy 178 | if not i % 1000: 179 | print i, vals, y 180 | if y < besty: 181 | bestvals, besty = vals, y 182 | 183 | return bestvals 184 | 185 | 186 | def score(producer, *vals): 187 | return evaluate_estimates(producer(*vals), True) 188 | -------------------------------------------------------------------------------- /timing_strats.py: -------------------------------------------------------------------------------- 1 | import random 2 | import hashlib 3 | import sys 4 | # Clock offset 5 | CLOCKOFFSET = 1 6 | # Block time 7 | BLKTIME = 40 8 | # Round run time 9 | ROUND_RUNTIME = 500000 10 | # Number of rounds 11 | ROUNDS = 2000 12 | # Block reward 13 | BLKREWARD = 1 14 | # Reward for including x ticks' worth of transactions 15 | # Linear by default, but sublinear formulas are 16 | # probably most accurate 17 | get_txreward = lambda ticks: 0.0001 * ticks 18 | # Latency function 19 | latency_sample = lambda L: int((random.expovariate(1) * ((L/1.33)**0.75))**1.33) 20 | # latency = lambda: random.randrange(15) if random.randrange(10) else 47 21 | # Offline rate 22 | OFFLINE_RATE = 0.03 23 | 24 | BLANK_STATE = {'transactions': 0} 25 | 26 | class Simulation(): 27 | def __init__(self, validators): 28 | for i, v in enumerate(validators): 29 | v.id = i 30 | v.simulation = self 31 | self.validators = validators 32 | self.time = 0 33 | self.next_id = 0 34 | self.gvcache = {} 35 | 36 | def run(self, rounds): 37 | # Run the simulation 38 | for i in range(rounds): 39 | for m in self.validators: 40 | m.mine() 41 | m.listen() 42 | self.time += 1 43 | if i % (rounds // 100) == 0: 44 | print 'Completed %d rounds out of %d' % (i, rounds) 45 | 46 | def get_validator(self, randao, skips): 47 | key = (randao << 32) + skips 48 | if key not in self.gvcache: 49 | self.gvcache[key] = sha256_as_int(key) % len(self.validators) 50 | return self.gvcache[key] 51 | 52 | 53 | class Block(): 54 | def __init__(self, parent, state, maker, number=0, skips=0): 55 | self.prevhash = parent.hash if parent else 0 56 | self.state = state 57 | self.number = number 58 | self.height = parent.height + 1 if parent else 0 59 | self.hash = random.randrange(10**20) + 10**23 * self.height 60 | self.randao = sha256_as_int((parent.randao if parent else 0) + maker) 61 | self.totskips = (parent.totskips if parent else 0) + skips 62 | 63 | def sha256_as_int(v): 64 | hashbytes = hashlib.sha256(str(v)).digest()[:4] 65 | o = 0 66 | for b in hashbytes: 67 | o = (o << 8) + ord(b) 68 | return o 69 | 70 | GENESIS = Block(None, BLANK_STATE, 0) 71 | 72 | # Insert a key/value pair into a state 73 | # This is abstracted away into a method to make it easier to 74 | # swap the state out with an immutable dict library or whatever 75 | # else to increase efficiency 76 | def update_state(s, k, v): 77 | s2 = {_k: _v for _k, _v in s.items()} 78 | s2[k] = v 79 | return s2 80 | 81 | # Get a key from a state, default zero 82 | def get_state(s, k): 83 | return s.get(k, 0) 84 | 85 | 86 | class Validator(): 87 | def __init__(self, strategy, latency=5): 88 | # The block that the validator considers to be the head 89 | self.head = GENESIS 90 | # A map of tick -> blocks that the validator will receive 91 | # during that tick 92 | self.listen_queue = {} 93 | # Blocks that the validator knows about 94 | self.blocks = {GENESIS.hash: GENESIS} 95 | # Scores (~= total subtree weight) for those blocks 96 | self.scores = {GENESIS.hash: 0} 97 | # When the validator received each block 98 | self.time_received = {GENESIS.hash: 0} 99 | # Received too early? 100 | self.received_too_early = {} 101 | # Blocks with missing parents, mapping parent hash -> list 102 | self.orphans_by_parent = {} 103 | # ... mapping hash -> list 104 | self.orphans = {} 105 | # This validator's clock is off by this number of ticks 106 | self.time_offset = random.randrange(CLOCKOFFSET) - CLOCKOFFSET // 2 107 | # Set the validator's strategy 108 | self.set_strategy(strategy) 109 | # Keeps track of the highest block number a validator has already 110 | # produced a block at 111 | self.min_number = 0 112 | # The validator's ID 113 | self.id = None 114 | # Blocks created 115 | self.created = 0 116 | # Number of blocks to backtrack for GHOST 117 | self.backtrack = 40 118 | # The simulation that this validator is in 119 | self.simulation = None 120 | # Maximum number of skips to try 121 | self.max_skips = 8 122 | # Network latency 123 | self.latency = latency 124 | 125 | def set_strategy(self, strategy): 126 | # The number of ticks a validator waits before producing a block 127 | self.produce_delay = strategy[0] 128 | # The number of extra ticks a validator waits per skip (ie. 129 | # if you skip two validator slots then wait this number of ticks 130 | # times two) before producing a block 131 | self.per_block_produce_delay = strategy[1] 132 | # The number of extra ticks a validator waits per skip before 133 | # accpeint a block 134 | self.per_block_accept_delay = strategy[2] 135 | 136 | 137 | # Get the time from the validator's point of view 138 | def get_time(self): 139 | return max(self.simulation.time + self.time_offset, 0) 140 | 141 | # Add a block to the listen queue at the given time 142 | def add_to_listen_queue(self, time, obj): 143 | if time not in self.listen_queue: 144 | self.listen_queue[time] = [] 145 | self.listen_queue[time].append(obj) 146 | 147 | def earliest_block_time(self, parent, skips): 148 | return 20 + parent.number * 20 + skips * 40 149 | 150 | def mine(self): 151 | # Is it time to produce a block? 152 | t = self.get_time() 153 | skips = 0 154 | head = self.head 155 | while self.simulation.get_validator(head.randao, skips) != self.id and skips < self.max_skips: 156 | skips += 1 157 | if skips == self.max_skips: 158 | return 159 | # If it is... 160 | if t >= self.time_received[head.hash] + self.produce_delay + self.per_block_produce_delay * skips \ 161 | and head.number >= self.min_number: 162 | # Can't produce a block at this height anymore 163 | self.min_number = head.number + 1 + skips 164 | # Small chance to be offline 165 | if random.random() < OFFLINE_RATE: 166 | return 167 | # Compute my block reward 168 | my_reward = BLKREWARD + get_txreward(self.simulation.time - head.state['transactions']) 169 | # Claim the reward from the transactions since the parent 170 | new_state = update_state(head.state, 'transactions', self.simulation.time) 171 | # Apply the block reward 172 | new_state = update_state(new_state, self.id, get_state(new_state, self.id) + my_reward) 173 | # Create the block 174 | b = Block(head, new_state, self.id, number=head.number + 1 + skips, skips=skips) 175 | self.created += 1 176 | # print '---------> Validator %d makes block with hash %d and parent %d (%d skips) at time %d' % (self.id, b.hash, b.prevhash, skips, self.simulation.time) 177 | # Broadcast it 178 | for validator in self.simulation.validators: 179 | recv_time = self.simulation.time + 1 + latency_sample(self.latency + validator.latency) 180 | # print 'broadcasting, delay %d' % (recv_time - t) 181 | validator.add_to_listen_queue(recv_time, b) 182 | 183 | # If a validator realizes that it "should" have a block but doesn't, 184 | # it can use this method to request it from the network 185 | def request_block(self, hash): 186 | for validator in self.simulation.validators: 187 | if hash in validator.blocks: 188 | recv_time = self.simulation.time + 1 + latency_sample(self.latency + validator.latency) 189 | self.add_to_listen_queue(recv_time, validator.blocks[hash]) 190 | 191 | # Process all blocks that it should receive during the current tick 192 | def listen(self): 193 | head = self.blocks[self.main_chain[-1]] 194 | if self.simulation.time in self.listen_queue: 195 | for blk in self.listen_queue[self.simulation.time]: 196 | self.accept_block(blk) 197 | if self.simulation.time in self.listen_queue: 198 | del self.listen_queue[self.simulation.time] 199 | 200 | def get_score_addition(self, blk): 201 | parent = self.blocks[blk.prevhash] 202 | skips = blk.number - parent.number - 1 203 | return (0 if blk.hash in self.received_too_early else 10**20) + random.randrange(100) - 50 204 | 205 | def accept_block(self, blk): 206 | t = self.get_time() 207 | # Parent not found or at least not yet processed 208 | if blk.prevhash not in self.blocks and blk.hash not in self.orphans: 209 | self.request_block(blk.prevhash) 210 | if blk.prevhash not in self.orphans_by_parent: 211 | self.orphans_by_parent[blk.prevhash] = [] 212 | self.orphans_by_parent[blk.prevhash].append(blk.hash) 213 | self.orphans[blk.hash] = blk 214 | # print 'validator %d skipping block %d: parent %d not found' % (self.id, blk.hash, blk.prevhash) 215 | return 216 | # Already processed? 217 | if blk.hash in self.blocks or blk.hash in self.orphans: 218 | # print 'validator %d skipping block %d: already processed' % (self.id, blk.hash) 219 | return 220 | # Too early? Re-append at earliest allowed time 221 | parent = self.blocks[blk.prevhash] 222 | skips = blk.number - parent.number - 1 223 | alotted_recv_time = self.time_received[parent.hash] + skips * self.per_block_accept_delay 224 | if t < alotted_recv_time: 225 | self.received_too_early[blk.hash] = alotted_recv_time - t 226 | self.add_to_listen_queue((alotted_recv_time - t) * 2 + self.simulation.time, blk) 227 | # print 'too early, validator %d delaying %d (%d vs %d)' % (self.id, blk.hash, t, alotted_recv_time) 228 | return 229 | # Add the block and compute the score 230 | # print 'Validator %d receives block, hash %d, time %d' % (self.id, blk.hash, self.simulation.time) 231 | self.blocks[blk.hash] = blk 232 | self.time_received[blk.hash] = t 233 | if blk.hash in self.orphans: 234 | del self.orphans[blk.hash] 235 | # Process the scoring rule 236 | self.head 237 | # print 'post', self.main_chain 238 | # self.scores[blk.hash] = self.scores[blk.prevhash] + get_score_addition(skips) 239 | if self.orphans_by_parent.get(blk.hash, []): 240 | for c in self.orphans_by_parent[blk.hash]: 241 | # print 'including previously rejected child of %d: %d' % (blk.hash, c) 242 | b = self.orphans[c] 243 | del self.orphans[c] 244 | self.accept_block(b) 245 | del self.orphans_by_parent[blk.hash] 246 | 247 | 248 | def simple_test(baseline=[10, 40, 31]): 249 | # Define the strategies of the validators 250 | strategy_groups = [ 251 | #((time before publishing a block, time per skip to wait before producing a block, time per skip to wait before accepting), latency, number of validators with this strategy) 252 | # ((baseline[0], baseline[1], baseline[2]), 12, 4), 253 | # ((baseline[0], baseline[1], baseline[2]), 11, 4), 254 | # ((baseline[0], baseline[1], baseline[2]), 10, 4), 255 | # ((baseline[0], baseline[1], baseline[2]), 9, 4), 256 | # ((baseline[0], baseline[1], baseline[2]), 8, 4), 257 | # ((baseline[0], baseline[1], baseline[2]), 7, 4), 258 | ((baseline[0], baseline[1], baseline[2]), 6, 5), 259 | ((baseline[0], baseline[1], baseline[2]), 5, 5), 260 | ((baseline[0], baseline[1], baseline[2]), 4, 5), 261 | ((baseline[0], baseline[1], baseline[2]), 3, 5), 262 | ((baseline[0], baseline[1], baseline[2]), 2, 5), 263 | ((baseline[0], baseline[1], baseline[2]), 1, 5), 264 | # ((baseline[0], int(baseline[1] * 0.33), baseline[2]), 3, 2), 265 | # ((baseline[0], int(baseline[1] * 0.67), baseline[2]), 3, 2), 266 | # ((baseline[0], int(baseline[1] * 1.5), baseline[2]), 3, 2), 267 | # ((baseline[0], int(baseline[1] * 2), baseline[2]), 3, 2), 268 | # ((baseline[0], baseline[1], int(baseline[2] * 0.33)), 3, 2), 269 | # ((baseline[0], baseline[1], int(baseline[2] * 0.67)), 3, 2), 270 | # ((baseline[0], baseline[1], int(baseline[2] * 1.5)), 3, 2), 271 | # ((baseline[0], baseline[1], int(baseline[2] * 2)), 3, 2), 272 | ] 273 | sgstarts = [0] 274 | 275 | validators = [] 276 | for s, l, c in strategy_groups: 277 | sgstarts.append(sgstarts[-1] + c) 278 | for i in range(c): 279 | validators.append(Validator(s, l)) 280 | 281 | Simulation(validators).run(ROUND_RUNTIME) 282 | 283 | def report(validators): 284 | head = validators[0].blocks[validators[0].main_chain[-1]] 285 | 286 | print 'Head block number:', head.number 287 | print 'Head block height:', head.height 288 | print head.state 289 | # print validators[0].scores 290 | 291 | for i, ((s, l, c), pos) in enumerate(zip(strategy_groups, sgstarts)): 292 | totrev = 0 293 | totcre = 0 294 | for j in range(pos, pos + c): 295 | totrev += head.state.get(j, 0) 296 | totcre += validators[j].created 297 | print 'Strategy group %d: average %d / %d / %d' % (i, totrev * 1.0 / c, totcre * 1.0 / c, (totrev * 2.0 - totcre) / c) 298 | 299 | report(validators) 300 | 301 | def evo_test(initial_s=[1, 40, 27]): 302 | s0 = [20, 40, 40] 303 | s = initial_s 304 | INITIAL_GROUP = 20 305 | DEVIATION_GROUP = 2 306 | LATENCY = 3 307 | for i in range(ROUNDS): 308 | print 's:', s, ', starting round', i 309 | strategy_groups = [ 310 | (s0, LATENCY, 1), 311 | (s, LATENCY, INITIAL_GROUP - 1), 312 | ] 313 | for j in range(len(s)): 314 | t = [x for x in s] 315 | t[j] = int(t[j] * 1.3) 316 | strategy_groups.append((t, LATENCY, DEVIATION_GROUP)) 317 | u = [x for x in s] 318 | u[j] = int(u[j] * 0.7) 319 | strategy_groups.append((u, LATENCY, DEVIATION_GROUP)) 320 | 321 | sgstarts = [0] 322 | 323 | validators = [] 324 | for _s, _l, c in strategy_groups: 325 | sgstarts.append(sgstarts[-1] + c) 326 | for i in range(c): 327 | validators.append(Validator(_s, _l)) 328 | 329 | Simulation(validators).run(ROUND_RUNTIME) 330 | head = validators[0].blocks[validators[0].main_chain[-1]] 331 | base_instate = sum([head.state.get(j, 0) for j in range(1, INITIAL_GROUP)]) * 1.0 / (INITIAL_GROUP - 1) 332 | base_totcreated = sum([validators[j].created for j in range(1, INITIAL_GROUP)]) * 1.0 / (INITIAL_GROUP - 1) 333 | base_reward = base_instate * 2 - base_totcreated 334 | print 'old s:', s 335 | for j in range(len(s)): 336 | L = INITIAL_GROUP + DEVIATION_GROUP * 2 * j 337 | M = INITIAL_GROUP + DEVIATION_GROUP * (2 * j + 1) 338 | R = INITIAL_GROUP + DEVIATION_GROUP * (2 * j + 2) 339 | up_instate = sum([head.state.get(k, 0) for k in range(L, M)]) * 1.0 / (M - L) 340 | up_totcreated = sum([validators[k].created for k in range(L, M)]) * 1.0 / (M - L) 341 | up_reward = up_instate * 2 - up_totcreated 342 | down_instate = sum([head.state.get(k, 0) for k in range(M, R)]) * 1.0 / (R - M) 343 | down_totcreated = sum([validators[k].created for k in range(M, R)]) * 1.0 / (R - M) 344 | down_reward = down_instate * 2 - down_totcreated 345 | print 'Adjusting variable %d: %.3f %.3f %.3f' % (j, down_reward, base_reward, up_reward) 346 | if up_reward > base_reward > down_reward: 347 | print 'increasing', j, s 348 | s[j] = int(s[j] * min(1 + (up_reward - base_reward) * 2. / base_reward, 1.2) + 0.49) 349 | elif down_reward > base_reward > up_reward: 350 | print 'decreasing', j, s 351 | s[j] = int(s[j] * max(1 - (down_reward - base_reward) * 2. / base_reward, 0.8) + 0.49) 352 | print 'new s:', s 353 | 354 | if len(sys.argv) >= 2 and sys.argv[1] == "evo": 355 | if len(sys.argv) > 4: 356 | evo_test([int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4])]) 357 | else: 358 | evo_test() 359 | elif len(sys.argv) >= 2 and sys.argv[1] == "onetime": 360 | if len(sys.argv) > 4: 361 | simple_test([int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4])]) 362 | else: 363 | simple_test() 364 | else: 365 | print 'Use evo or onetime' 366 | --------------------------------------------------------------------------------