├── .gitignore
├── NPoS
    ├── .gitignore
    ├── setup.cfg
    ├── remote_test.py
    ├── auctiondynamicthing.py
    ├── npos.py
    └── ComplicatedPhragmén.py
├── pdf
    ├── cc-by.pdf
    ├── grandpa.pdf
    ├── grandpa-old.pdf
    ├── lipics-logo-bw.pdf
    ├── .gitignore
    ├── grandpa.bib
    ├── theory.bib
    └── grandpa.tex
├── README.md
├── random
    └── randao_analysis.ipynb
└── Polkadot_Finality_Gadget_v9000.md


/.gitignore:
--------------------------------------------------------------------------------
1 | .vscode
2 | 


--------------------------------------------------------------------------------
/NPoS/.gitignore:
--------------------------------------------------------------------------------
1 | .vscode
2 | __pycache__
3 | 


--------------------------------------------------------------------------------
/NPoS/setup.cfg:
--------------------------------------------------------------------------------
1 | [flake8]
2 | max-line-length = 120
3 | 


--------------------------------------------------------------------------------
/pdf/cc-by.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/w3f/consensus/HEAD/pdf/cc-by.pdf


--------------------------------------------------------------------------------
/pdf/grandpa.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/w3f/consensus/HEAD/pdf/grandpa.pdf


--------------------------------------------------------------------------------
/pdf/grandpa-old.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/w3f/consensus/HEAD/pdf/grandpa-old.pdf


--------------------------------------------------------------------------------
/pdf/lipics-logo-bw.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/w3f/consensus/HEAD/pdf/lipics-logo-bw.pdf


--------------------------------------------------------------------------------
/pdf/.gitignore:
--------------------------------------------------------------------------------
 1 | # Add any directories, files, or patterns you don't want to be tracked by version control
 2 | 
 3 | *.log
 4 | *.aux
 5 | *.synctex.gz
 6 | *.bbl
 7 | *.blg
 8 | *.swp
 9 | *.fls
10 | *.fdb_latexmk
11 | *.out
12 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Overview
2 | 
3 | Repository for keeping track of research, specifications and implementations of consensus related algorithms. All algorithms are developed by or in collaboration with the Web3 Foundation.
4 | 
5 | ## Projects
6 | 
7 | - Polkadot Finality Gadget (AFG)
8 | 


--------------------------------------------------------------------------------
/NPoS/remote_test.py:
--------------------------------------------------------------------------------
 1 | import npos
 2 | import pprint
 3 | from substrateinterface import SubstrateInterface
 4 | from substrateinterface.utils.ss58 import ss58_decode, ss58_encode
 5 | 
 6 | 
 7 | pp = pprint.PrettyPrinter(indent=4)
 8 | 
 9 | substrate = SubstrateInterface(
10 |     url="ws://localhost:9944",
11 |     address_type=0,
12 |     type_registry={'types': {
13 |             "StakingLedger<AccountId, BalanceOf>": {
14 |                 "type": "struct",
15 |                 "type_mapping": [
16 |                     ["stash", "AccountId"],
17 |                     ["total", "Compact<Balance>"],
18 |                     ["active", "Compact<Balance>"],
19 |                     ["unlocking", "Vec<UnlockChunk<Balance>>"],
20 |                     ["claimedReward", "Vec<EraIndex>"]
21 |                 ]
22 |             },
23 |         }
24 |     },
25 |     type_registry_preset='polkadot',
26 | )
27 | 
28 | head = substrate.get_chain_finalised_head()
29 | 
30 | 
31 | def get_candidates():
32 |     prefix = substrate.generate_storage_hash("Staking", "Validators")
33 |     pairs = substrate.rpc_request(method="state_getPairs", params=[prefix, head])['result']
34 |     last_32_bytes = list(map(lambda p: "0x" + p[0][-64:], pairs))
35 |     return list(map(lambda k: ss58_encode(k), last_32_bytes))
36 | 
37 | 
38 | def get_nominators():
39 |     prefix = substrate.generate_storage_hash("Staking", "Nominators")
40 |     pairs = substrate.rpc_request(method="state_getPairs", params=[prefix, head])['result']
41 | 
42 |     nominators = list(
43 |         map(lambda p: ("0x" + p[0][-64:], substrate.decode_scale("Nominations<AccountId>", p[1])['targets']), pairs)
44 |     )
45 | 
46 |     nominators = list(map(lambda x: (
47 |         ss58_encode(x[0], substrate.address_type),
48 |         x[1],
49 |     ), nominators))
50 | 
51 |     return list(map(lambda x: (
52 |         x[0],
53 |         get_backing_stake_of(x[0]),
54 |         [ss58_encode(acc, substrate.address_type) for acc in x[1]],
55 |     ), nominators))
56 | 
57 | 
58 | def get_backing_stake_of(who):
59 |     ctrl = substrate.get_runtime_state(
60 |         module="Staking",
61 |         storage_function="Bonded",
62 |         params=[who],
63 |         block_hash=head,
64 |     )['result']
65 | 
66 |     ctrl = ss58_encode(ctrl, substrate.address_type)
67 | 
68 |     ledger = substrate.get_runtime_state(
69 |         module="Staking",
70 |         storage_function="Ledger",
71 |         params=[ctrl],
72 |         block_hash=head,
73 |     )['result']
74 | 
75 |     return ledger['active']
76 | 
77 | 
78 | def validator_count():
79 |     return substrate.get_runtime_state("Staking", "ValidatorCount", [], head)['result']
80 | 
81 | 
82 | candidates = get_candidates()
83 | nominators = get_nominators()
84 | to_elect = validator_count()
85 | 
86 | print("{} validators, {} nominators, electing {}".format(len(candidates), len(nominators), to_elect))
87 | distribution, winners = npos.phragmms(nominators, to_elect)
88 | npos.print_list(winners)
89 | print(sum([c.backed_stake for c in winners]))
90 | print(min([c.backed_stake for c in winners]))
91 | score = [min([c.backed_stake for c in winners]), sum([c.backed_stake for c in winners])]
92 | print("score = [{:,}, {:,}]".format(int(score[0]), int(score[1])))
93 | 


--------------------------------------------------------------------------------
/NPoS/auctiondynamicthing.py:
--------------------------------------------------------------------------------
 1 | class auction:
 2 |     def __init__(self):
 3 |         #Note that ranges are intervals not slices;
 4 |         #the lenth 1 range has start=end not end=start+1
 5 |         self.ranges = [(start,end) for end in range(4) for start in range(end+1)]
 6 |         self.winningbidsonranges=[(None,0) for _ in self.ranges]
 7 |     def bid(self,bidid,start,end,amount,verbose=False):
 8 |         if verbose:
 9 |             print(bidid," bids ",amount," on [",start,",",end,"]")
10 |         #Bid should intersect with all winning bids by same bidder
11 |         for rangeotherbid in [i for i,w in zip(self.ranges,self.winningbidsonranges) if w[0]==bidid]:
12 |             if rangeotherbid[0] > end + 1 or rangeotherbid[1] + 1  < start:
13 |                 if verbose:
14 |                     print("Invalid bid as winning on [",rangeotherbid[0],",",rangeotherbid[1],"]")
15 |                 return
16 |         rangeindex=self.ranges.index((start,end))
17 |         _,currentwinningbid=self.winningbidsonranges[rangeindex]
18 |         if amount > currentwinningbid:
19 |             self.winningbidsonranges[rangeindex] = (bidid,amount)
20 |     def bestbidonrangeandvalue(self,start,end):
21 |         rangeindex=self.ranges.index((start,end))
22 |         return ((self.winningbidsonranges[rangeindex][0],start,end), self.winningbidsonranges[rangeindex][1]*(end-start+1))
23 |     def calculatewinners(self):
24 |         bestwinnersendingat=[([],0) for _ in range(4)]
25 |         for i in range(4):
26 |             bidderrange,value=self.bestbidonrangeandvalue(0,i)
27 |             bestwinnersendingat[i]=([bidderrange],value)
28 |             for j in range(i):
29 |                 bidderrange,value = self.bestbidonrangeandvalue(j+1,i)
30 |                 value += bestwinnersendingat[j][1]
31 |                 if value > bestwinnersendingat[i][1]:
32 |                     newwinners=bestwinnersendingat[j][0]+[bidderrange]
33 |                     bestwinnersendingat[i]=(newwinners,value)
34 |         return bestwinnersendingat[3]
35 | def example1():
36 |     a=auction()
37 |     a.bid("x",0,3,1,True)
38 |     a.bid("a",0,0,2,True)
39 |     a.bid("a",2,2,53,True)
40 |     a.bid("b",1,1,1,True)
41 |     a.bid("c",2,2,1,True)
42 |     a.bid("d",3,3,1,True)
43 |     print(a.calculatewinners())
44 |     a.bid("x",0,3,2,True)
45 |     print(a.calculatewinners())
46 | 
47 | import unittest
48 | class AuctionTests(unittest.TestCase):
49 |     def testbids(self):
50 |         a=auction();
51 |         #We could even allow zero bids but we don't now:
52 |         a.bid("a",0,3,0, False)
53 |         self.assertIsNone(a.bestbidonrangeandvalue(0,3)[0][0])
54 |         a.bid("a",0,3,1, False)
55 |         self.assertEqual(a.bestbidonrangeandvalue(0,3)[0][0],"a")
56 |         #Overlapping bids are fine
57 |         a.bid("a",0,0,1, False)
58 |         self.assertEqual(a.bestbidonrangeandvalue(0,0)[0][0],"a")
59 |         # Separated bids are not
60 |         a.bid("a",2,2,1, False)
61 |         self.assertIsNone(a.bestbidonrangeandvalue(2,2)[0][0])
62 |         # Contiguous can be
63 |         a.bid("a",1,1,1, False)
64 |         self.assertEqual(a.bestbidonrangeandvalue(1,1)[0][0],"a")
65 |         #Equal bids don't win
66 |         a.bid("b",0,3,1, False)
67 |         self.assertEqual(a.bestbidonrangeandvalue(0,3)[0][0],"a")
68 |         #Higher bids do
69 |         a.bid("b",0,3,2, False)
70 |         self.assertEqual(a.bestbidonrangeandvalue(0,3)[0][0],"b")
71 |         
72 |         
73 |         
74 |     def testExample1(self):
75 |         a=auction()
76 |         a.bid("x",0,3,1)
77 |         a.bid("a",0,0,2)
78 |         a.bid("a",2,2,53)
79 |         a.bid("b",1,1,1)
80 |         a.bid("c",2,2,1)
81 |         a.bid("d",3,3,1)
82 |         self.assertEqual(a.calculatewinners(), ([('a', 0, 0), ('b', 1, 1), ('c', 2, 2), ('d', 3, 3)], 5))
83 |         a.bid("x",0,3,2)
84 |         self.assertEqual(a.calculatewinners(),([('x', 0, 3)], 8))
85 | 
86 |         
87 |         
88 |         
89 |         
90 |         
91 |         
92 |         
93 |     
94 | 
95 | 


--------------------------------------------------------------------------------
/pdf/grandpa.bib:
--------------------------------------------------------------------------------
  1 | @article{flp,
  2 |   title={Impossibility of distributed consensus with one faulty process},
  3 |   author={Fischer, Michael J and Lynch, Nancy A and Paterson, Michael S},
  4 |   journal={Journal of the ACM (JACM)},
  5 |   volume={32},
  6 |   number={2},
  7 |   pages={374--382},
  8 |   year={1985},
  9 |   publisher={ACM},
 10 |   url={https://groups.csail.mit.edu/tds/papers/Lynch/jacm85.pdf}
 11 | }
 12 | 
 13 | @article{CasperFFG,
 14 |   title={Casper the friendly finality gadget},
 15 |   author={Buterin, Vitalik and Griffith, Virgil},
 16 |   journal={arXiv preprint arXiv:1710.09437},
 17 |   year={2017},
 18 |   url={https://arxiv.org/abs/1710.09437}
 19 | }
 20 | 
 21 | @article{Tendermint,
 22 |   title={The latest gossip on BFT consensus},
 23 |   author={Buchman, Ethan and Kwon, Jae and Milosevic, Zarko},
 24 |   journal={arXiv preprint arXiv:1807.04938},
 25 |   year={2018},
 26 |   url={https://arxiv.org/abs/1807.04938}
 27 | }
 28 | 
 29 | @article{CasperCBC,
 30 |   title={Casper the Friendly Ghost: A “Correct-by-Construction” Blockchain Consensus Protocol},
 31 |   author={Zamfir,Vlad},
 32 |   year={2017},
 33 |   url={https://github.com/ethereum/research/blob/master/papers/CasperTFG/CasperTFG.pdf}
 34 | }
 35 | 
 36 | 
 37 | @article{BitcoinBA,
 38 |   title={Anonymous byzantine consensus from moderately-hard puzzles: A model for bitcoin},
 39 |   author={Miller, Andrew and LaViola Jr, Joseph J},
 40 |   url={https://nakamotoinstitute.org/research/anonymous-byzantine-consensus/},
 41 |   year={2014}
 42 | }
 43 | 
 44 | @inproceedings{Discoin,
 45 |   title={Bitcoin meets strong consistency},
 46 |   author={Decker, Christian and Seidel, Jochen and Wattenhofer, Roger},
 47 |   booktitle={Proceedings of the 17th International Conference on Distributed Computing and Networking},
 48 |   pages={13},
 49 |   year={2016},
 50 |   organization={ACM},
 51 |   url={https://arxiv.org/abs/1412.7935}
 52 | }
 53 | 
 54 | @article{SCP,
 55 |   title={SCP: A Computationally-Scalable Byzantine Consensus Protocol For Blockchains.},
 56 |   author={Luu, Loi and Narayanan, Viswesh and Baweja, Kunal and Zheng, Chaodong and Gilbert, Seth and Saxena, Prateek},
 57 |   journal={IACR Cryptology ePrint Archive},
 58 |   volume={2015},
 59 |   pages={1168},
 60 |   year={2015},
 61 |   url={https://www.weusecoins.com/assets/pdf/library/SCP%20-%20%20A%20Computationally-Scalable%20Byzantine.pdf}
 62 | }
 63 | 
 64 | @article{DLS,
 65 |   title={Consensus in the presence of partial synchrony},
 66 |   author={Dwork, Cynthia and Lynch, Nancy and Stockmeyer, Larry},
 67 |   journal={Journal of the ACM (JACM)},
 68 |   volume={35},
 69 |   number={2},
 70 |   pages={288--323},
 71 |   year={1988},
 72 |   publisher={ACM New York, NY, USA}
 73 | }
 74 | 
 75 | @misc{ nakamoto08bitcoin,
 76 |   author = {Satoshi Nakamoto},
 77 |   title = {\href{https://bitcoin.org/bitcoin.pdf}{Bitcoin: A Peer-to-Peer Electronic Cash System}},
 78 |   year = 2008,
 79 | }
 80 | 
 81 | @article{wood14ethereum,
 82 |   title={\href{https://github.com/ethereum/wiki/wiki/White-Paper}{Ethereum: A Secure Decentralised Generalised Transaction Ledger}},
 83 |   author={Wood, Gavin},
 84 |   journal={Ethereum Project Yellow Paper},
 85 |   year={2014}
 86 | }
 87 | 
 88 | @inproceedings{sasson2014zerocash,
 89 |   title={\href{https://ieeexplore.ieee.org/abstract/document/6956581/}{Zerocash: Decentralized anonymous payments from {B}itcoin}},
 90 |   author={Sasson, Eli Ben and Chiesa, Alessandro and Garman, Christina and Green, Matthew and Miers, Ian and Tromer, Eran and Virza, Madars},
 91 |   booktitle={Security and Privacy (SP), 2014 IEEE Symposium on},
 92 |   pages={459--474},
 93 |   year={2014},
 94 |   organization={IEEE}
 95 | }
 96 | 
 97 | @article{apostolaki16hijacking,
 98 |   title={\href{http://arxiv.org/abs/1605.07524}{Hijacking Bitcoin: Large-scale Network Attacks on Cryptocurrencies}},
 99 |   author={Apostolaki, Maria and Zohar, Aviv and Vanbever, Laurent},
100 |   journal = {38th IEEE Symposium on Security and Privacy},
101 |   month = may,
102 |   year = {2017}
103 | }
104 | 
105 | @inproceedings{gervais15tampering,
106 |   title={\href{https://eprint.iacr.org/2015/578.pdf}{Tampering with the Delivery of Blocks and Transactions in {Bitcoin}}},
107 |   author={Gervais, Arthur and Ritzdorf, Hubert and Karame, Ghassan O and Capkun, Srdjan},
108 |   booktitle={22nd ACM SIGSAC Conference on Computer and Communications Security},
109 |   pages={692--705},
110 |   year={2015},
111 |   organization={ACM},
112 |   url = {https://eprint.iacr.org/2015/578.pdf},
113 | }
114 | 
115 | @inproceedings{heilman15eclipse,
116 |   title={\href{https://www.usenix.org/system/files/conference/usenixsecurity15/sec15-paper-heilman.pdf}{Eclipse Attacks on {Bitcoin}'s Peer-to-Peer Network}},
117 |   author={Heilman, Ethan and Kendler, Alison and Zohar, Aviv and Goldberg, Sharon},
118 |   booktitle={24th USENIX Security Symposium},
119 |   pages={129--144},
120 |   year={2015},
121 |   url={https://www.usenix.org/system/files/conference/usenixsecurity15/sec15-paper-heilman.pdf},
122 | }
123 | 
124 | @inproceedings{kokoris16enhancing,
125 |   author        = {Eleftherios Kokoris-Kogias and Philipp Jovanovic and Nicolas Gailly and Ismail Khoffi and Linus Gasser and Bryan Ford},
126 |   title         = {\href{http://arxiv.org/abs/1602.06997}{Enhancing Bitcoin Security and Performance with Strong Consistency via Collective Signing}},
127 |   booktitle     = {Proceedings of the 25th USENIX Conference on Security Symposium},
128 |   year          = {2016},
129 | }
130 | 
131 | @InProceedings{decker16bitcoin,
132 |   author =  {Christian Decker and Jochen Seidel and Roger Wattenhofer},
133 |   title = {\href{http://www.tik.ee.ethz.ch/file/ed3e5da74fbca5584920e434d9976a12/peercensus.pdf}{Bitcoin Meets Strong Consistency}},
134 |   booktitle = {{17th International Conference on Distributed Computing and Networking (ICDCN), Singapore}},
135 |   month = {January},
136 |   year =    {2016},
137 |   url =   {http://www.tik.ee.ethz.ch/file/ed3e5da74fbca5584920e434d9976a12/peercensus.pdf},
138 | }
139 | 
140 | @misc{pass16hybrid,
141 |     author = {Rafael Pass and Elaine Shi},
142 |     title = {\href{http://eprint.iacr.org/2016/917}{Hybrid Consensus: Efficient Consensus in the Permissionless Model}},
143 |     howpublished = {Cryptology ePrint Archive, Report 2016/917},
144 |     year = {2016},
145 | }
146 | 
147 | @article{avarikioti19divide,
148 |   title={Divide and Scale: Formalization of Distributed Ledger Sharding Protocols},
149 |   author={Avarikioti, Georgia and Kokoris-Kogias, Eleftherios and Wattenhofer, Roger},
150 |   journal={arXiv preprint arXiv:1910.10434},
151 |   year={2019}
152 | }
153 | 
154 | @inproceedings{kokoris17omniledger,
155 |   title={\href{https://eprint.iacr.org/2017/406.pdf}{OmniLedger: A Secure, Scale-Out, Decentralized Ledger via Sharding}},
156 |   author={Kokoris-Kogias, Eleftherios and Jovanovic, Philipp and Gasser, Linus and Gailly, Nicolas and Syta, Ewa and Ford, Bryan},
157 |   booktitle={39th {IEEE Symposium on Security and Privacy}},
158 |   pages={19--34},
159 |   year={2018},
160 |   organization={IEEE}
161 | }
162 | 
163 | @inproceedings{al18chainspace,
164 |   author    = {Mustafa Al{-}Bassam and Alberto Sonnino and Shehar Bano and
165 |                Dave Hrycyszyn and George Danezis},
166 |   title     = {\href{https://arxiv.org/pdf/1708.03778.pdf}{Chainspace: {A} Sharded Smart Contracts Platform}},
167 |   booktitle = {25th Annual Network and Distributed System Security Symposium, {NDSS}
168 |                2018, San Diego, California, USA, February 18-21, 2018},
169 |   year      = {2018},
170 | }
171 | 
172 | @inproceedings{androulaki18channels,
173 |   title={\href{https://link.springer.com/chapter/10.1007/978-3-319-99073-6_6}{Channels: Horizontal Scaling and Confidentiality on Permissioned Blockchains}},
174 |   author={Androulaki, Elli and Cachin, Christian and De Caro, Angelo and Kokoris-Kogias, Eleftherios},
175 |   booktitle={European Symposium on Research in Computer Security},
176 |   pages={111--131},
177 |   year={2018},
178 |   organization={Springer}
179 | }
180 | 
181 | @techreport{zamyatin19sok,
182 |   title={Sok: Communication across distributed ledgers},
183 |   author={Zamyatin, Alexei and Al-Bassam, Mustafa and Zindros, Dionysis and Kokoris-Kogias, Eleftherios and Moreno-Sanchez, Pedro and Kiayias, Aggelos and Knottenbelt, William J},
184 |   year={2019},
185 |   institution={IACR Cryptology ePrint Archive, 2019: 1128}
186 | }
187 | 
188 | @inproceedings{lewenberg15inclusive,
189 |   title={\href{https://link.springer.com/chapter/10.1007/978-3-662-47854-7_33}{Inclusive block chain protocols}},
190 |   author={Lewenberg, Yoad and Sompolinsky, Yonatan and Zohar, Aviv},
191 |   booktitle={International Conference on Financial Cryptography and Data Security},
192 |   pages={528--547},
193 |   year={2015},
194 |   organization={Springer}
195 | }
196 | 


--------------------------------------------------------------------------------
/NPoS/npos.py:
--------------------------------------------------------------------------------
  1 | import unittest
  2 | 
  3 | 
  4 | def print_list(ll):
  5 |     for item in ll:
  6 |         print(item)
  7 | 
  8 | 
  9 | class edge:
 10 |     def __init__(self, nominator_id, validator_id):
 11 |         self.nominator_id = nominator_id
 12 |         self.validator_id = validator_id
 13 |         self.load = 0
 14 |         self.weight = 0
 15 |         self.candidate = None
 16 | 
 17 |     def __str__(self):
 18 |         return "Edge({}, weight = {:,})".format(
 19 |             self.validator_id,
 20 |             self.weight,
 21 |         )
 22 | 
 23 | 
 24 | class nominator:
 25 |     def __init__(self, nominator_id, budget, targets):
 26 |         self.nominator_id = nominator_id
 27 |         self.budget = budget
 28 |         self.edges = [edge(self.nominator_id, validator_id) for validator_id in targets]
 29 |         self.load = 0
 30 | 
 31 |     def __str__(self):
 32 |         return "Nominator({}, budget = {:,}, load = {}, edges = {})".format(
 33 |             self.nominator_id,
 34 |             self.budget,
 35 |             self.load,
 36 |             [str(e) for e in self.edges]
 37 |         )
 38 | 
 39 | 
 40 | class candidate:
 41 |     def __init__(self, validator_id, index):
 42 |         self.validator_id = validator_id
 43 |         self.valindex = index
 44 |         self.approval_stake = 0
 45 |         self.backed_stake = 0
 46 |         self.elected = False
 47 |         self.score = 0
 48 |         self.scoredenom = 0
 49 | 
 50 |     def __str__(self):
 51 |         return "Candidate({}, approval = {:,}, backed_stake = {:,})".format(
 52 |             self.validator_id,
 53 |             self.approval_stake,
 54 |             int(self.backed_stake),
 55 |         )
 56 | 
 57 | 
 58 | def seq_phragmen(votelist, num_to_elect):
 59 |     nomlist, candidates = setuplists(votelist)
 60 |     calculate_approval(nomlist)
 61 | 
 62 |     elected_candidates = list()
 63 |     for round in range(num_to_elect):
 64 |         for candidate in candidates:
 65 |             if not candidate.elected:
 66 |                 candidate.score = 1/candidate.approval_stake
 67 |         for nom in nomlist:
 68 |             for edge in nom.edges:
 69 |                 if not edge.candidate.elected:
 70 |                     edge.candidate.score += nom.budget * nom.load / edge.candidate.approval_stake
 71 |         best_candidate = 0
 72 |         best_score = 1000  # should be infinite but I'm lazy
 73 |         for candidate in candidates:
 74 |             if not candidate.elected and candidate.score < best_score:
 75 |                 best_score = candidate.score
 76 |                 best_candidate = candidate.valindex
 77 |         elected_candidate = candidates[best_candidate]
 78 |         elected_candidate.elected = True
 79 |         elected_candidate.electedpos = round
 80 |         elected_candidates.append(elected_candidate)
 81 |         for nom in nomlist:
 82 |             for edge in nom.edges:
 83 |                 if edge.candidate.valindex == best_candidate:
 84 |                     edge.load = elected_candidate.score - nom.load
 85 |                     nom.load = elected_candidate.score
 86 | 
 87 |     for candidate in elected_candidates:
 88 |         candidate.backed_stake = 0
 89 | 
 90 |     for nom in nomlist:
 91 |         for edge in nom.edges:
 92 |             if nom.load > 0.0:
 93 |                 edge.weight = nom.budget * edge.load/nom.load
 94 |                 edge.candidate.backed_stake += edge.weight
 95 |             else:
 96 |                 edge.weight = 0
 97 |     return (nomlist, elected_candidates)
 98 | 
 99 | 
100 | def equalise(nom, tolerance):
101 |     # Attempts to redistribute the nominators budget between elected validators. Assumes that all
102 |     # elected validators have backed_stake set correctly. Returns the max difference in stakes
103 |     # between sup.
104 | 
105 |     elected_edges = [edge for edge in nom.edges if edge.candidate.elected]
106 | 
107 |     if len(elected_edges) < 2:
108 |         return 0.0
109 | 
110 |     stake_used = sum([edge.weight for edge in elected_edges])
111 |     backed_stakes = [edge.candidate.backed_stake for edge in elected_edges]
112 |     backingbacked_stakes = [
113 |         edge.candidate.backed_stake for edge in elected_edges if edge.weight > 0.0
114 |     ]
115 | 
116 |     if len(backingbacked_stakes) > 0:
117 |         difference = max(backingbacked_stakes)-min(backed_stakes)
118 |         difference += nom.budget - stake_used
119 |         if difference < tolerance:
120 |             return difference
121 |     else:
122 |         difference = nom.budget
123 | 
124 |     # remove all backing
125 |     for edge in nom.edges:
126 |         edge.candidate.backed_stake -= edge.weight
127 |         edge.weight = 0
128 | 
129 |     elected_edges.sort(key=lambda x: x.candidate.backed_stake)
130 |     cumulative_backed_stake = 0
131 |     last_index = len(elected_edges) - 1
132 | 
133 |     for i in range(len(elected_edges)):
134 |         backed_stake = elected_edges[i].candidate.backed_stake
135 |         if backed_stake * i - cumulative_backed_stake > nom.budget:
136 |             last_index = i-1
137 |             break
138 |         cumulative_backed_stake += backed_stake
139 | 
140 |     last_stake = elected_edges[last_index].candidate.backed_stake
141 |     ways_to_split = last_index+1
142 |     excess = nom.budget + cumulative_backed_stake - last_stake*ways_to_split
143 | 
144 |     for edge in elected_edges[0:ways_to_split]:
145 |         edge.weight = excess / ways_to_split + last_stake - edge.candidate.backed_stake
146 |         edge.candidate.backed_stake += edge.weight
147 | 
148 |     return difference
149 | 
150 | 
151 | def equalise_all(nomlist, maxiterations, tolerance):
152 |     for i in range(maxiterations):
153 |         # for j in range(len(nomlist)):
154 |         #     nom = random.choice(nomlist)
155 |         #     equalise(nom, tolerance)
156 |         maxdifference = 0
157 |         for nom in nomlist:
158 |             difference = equalise(nom, tolerance)
159 |             maxdifference = max(difference, maxdifference)
160 |         if maxdifference < tolerance:
161 |             return
162 | 
163 | 
164 | def seq_phragmen_with_equalise(votelist, num_to_elect):
165 |     nomlist, elected_candidates = seq_phragmen(votelist, num_to_elect)
166 |     equalise_all(nomlist, 2, 0)
167 |     return nomlist, elected_candidates
168 | 
169 | 
170 | def calculateMaxScoreNoCutoff(nomlist, candidates):
171 |     # First we compute the denominator of the score
172 |     for candidate in candidates:
173 |         if not candidate.elected:
174 |             candidate.scoredenom = 1.0
175 | 
176 |     for nom in nomlist:
177 |         denominator_contrib = 0
178 | 
179 |         for edge in nom.edges:
180 |             if edge.candidate.elected:
181 |                 denominator_contrib += edge.weight/edge.candidate.backed_stake
182 | 
183 |         for edge in nom.edges:
184 |             if not edge.candidate.elected:
185 |                 edge.candidate.scoredenom += denominator_contrib
186 | 
187 |     # Then we divide. Not that score here is comparable to the recipricol of the score in
188 |     # seq-phragmen. In particular there low scores are good whereas here high scores are good.
189 |     best_candidate = 0
190 |     best_score = 0.0
191 |     for candidate in candidates:
192 |         if candidate.approval_stake > 0.0:
193 |             candidate.score = candidate.approval_stake / candidate.scoredenom
194 |             if not candidate.elected and candidate.score > best_score:
195 |                 best_score = candidate.score
196 |                 best_candidate = candidate
197 |         else:
198 |             candidate.score = 0.0
199 | 
200 |     return (best_candidate, best_score)
201 | 
202 | 
203 | def electWithScore(nomlist, elected_candidate, cutoff):
204 |     for nom in nomlist:
205 |         for new_edge in nom.edges:
206 |             if new_edge.validator_id == elected_candidate.validator_id:
207 |                 used_budget = sum([edge.weight for edge in nom.edges])
208 | 
209 |                 new_edge.weight = nom.budget - used_budget
210 |                 elected_candidate.backed_stake += nom.budget - used_budget
211 | 
212 |                 for edge in nom.edges:
213 |                     if edge.validator_id != elected_candidate.validator_id and edge.weight > 0.0:
214 |                         if edge.candidate.backed_stake > cutoff:
215 |                             stake_to_take = edge.weight * cutoff / edge.candidate.backed_stake
216 | 
217 |                             new_edge.weight += stake_to_take
218 |                             elected_candidate.backed_stake += stake_to_take
219 | 
220 |                             edge.weight -= stake_to_take
221 |                             edge.candidate.backed_stake -= stake_to_take
222 | 
223 | 
224 | def phragmms(votelist, num_to_elect, tolerance=0.1):
225 |     nomlist, candidates = setuplists(votelist)
226 |     calculate_approval(nomlist)
227 | 
228 |     elected_candidates = list()
229 |     for round in range(num_to_elect):
230 |         (elected_candidate, score) = calculateMaxScoreNoCutoff(nomlist, candidates)
231 |         electWithScore(nomlist, elected_candidate, score)
232 | 
233 |         elected_candidate.elected = True
234 |         elected_candidates.append(elected_candidate)
235 |         elected_candidate.electedpos = round
236 | 
237 |         equalise_all(nomlist, 10, tolerance)
238 | 
239 |     return nomlist, elected_candidates
240 | 
241 | 
242 | def approval_voting(votelist, num_to_elect):
243 |     nomlist, candidates = setuplists(votelist)
244 |     # Compute the total possible stake for each candidate
245 |     for nom in nomlist:
246 |         for edge in nom.edges:
247 |             edge.candidate.approval_stake += nom.budget
248 |             edge.weight = nom.budget/min(len(nom.edges), num_to_elect)
249 |             edge.candidate.backed_stake += edge.weight
250 |     candidates.sort(key=lambda x: x.approval_stake, reverse=True)
251 |     elected_candidates = candidates[0:num_to_elect]
252 |     return nomlist, elected_candidates
253 | 
254 | 
255 | def calculate_approval(nomlist):
256 |     for nom in nomlist:
257 |         for edge in nom.edges:
258 |             edge.candidate.approval_stake += nom.budget
259 | 
260 | 
261 | def setuplists(votelist):
262 |     '''
263 |     Basically populates edge.candidate, and returns nomlist and candidate array. The former is a
264 |     flat list of nominators and the latter is a flat list of validator candidates.
265 | 
266 |     Instead of Python's dict here, you can use anything with O(log n) addition and lookup. We can
267 |     also use a hashmap like dict, by generating a random constant r and using H(canid+r) since the
268 |     naive thing is obviously attackable.
269 |     '''
270 |     nomlist = [nominator(votetuple[0], votetuple[1], votetuple[2]) for votetuple in votelist]
271 |     # Basically used as a cache.
272 |     candidate_dict = dict()
273 |     candidate_array = list()
274 |     num_candidates = 0
275 |     # Get an array of candidates.# We could reference these by index rather than pointer
276 |     for nom in nomlist:
277 |         for edge in nom.edges:
278 |             validator_id = edge.validator_id
279 |             if validator_id in candidate_dict:
280 |                 index = candidate_dict[validator_id]
281 |                 edge.candidate = candidate_array[index]
282 |             else:
283 |                 candidate_dict[validator_id] = num_candidates
284 |                 newcandidate = candidate(validator_id, num_candidates)
285 |                 candidate_array.append(newcandidate)
286 | 
287 |                 edge.candidate = newcandidate
288 |                 num_candidates += 1
289 |     return nomlist, candidate_array
290 | 
291 | 
292 | def run_and_print_all(votelist, to_elect):
293 |     print("######\nVotes ", votelist)
294 | 
295 |     print("\nSequential Phragmén gives")
296 |     nomlist, elected_candidates = seq_phragmen(votelist, to_elect)
297 |     printresult(nomlist, elected_candidates)
298 | 
299 |     print("\nApproval voting gives")
300 |     nomlist, elected_candidates = approval_voting(votelist, to_elect)
301 |     printresult(nomlist, elected_candidates)
302 | 
303 |     print("\nSequential Phragmén with post processing gives")
304 |     nomlist, elected_candidates = seq_phragmen_with_equalise(votelist, to_elect)
305 |     printresult(nomlist, elected_candidates)
306 | 
307 |     print("\nBalanced Heuristic (3.15 factor) gives")
308 |     nomlist, elected_candidates = phragmms(votelist, to_elect)
309 |     printresult(nomlist, elected_candidates)
310 | 
311 | 
312 | def printresult(nomlist, elected_candidates, verbose=True):
313 |     for candidate in elected_candidates:
314 |         print(candidate.validator_id, " is elected with stake ",
315 |               candidate.backed_stake, "and score ", candidate.score)
316 |     if verbose:
317 |         for nom in nomlist:
318 |             print(nom.nominator_id, " has load ", nom.load, "and supported ")
319 |             for edge in nom.edges:
320 |                 print(edge.validator_id, " with stake ", edge.weight, end=", ")
321 |             print()
322 |         print()
323 | 
324 | 
325 | def example1():
326 |     votelist = [
327 |         ("A", 10.0, ["X", "Y"]),
328 |         ("B", 20.0, ["X", "Z"]),
329 |         ("C", 30.0, ["Y", "Z"]),
330 |     ]
331 |     run_and_print_all(votelist, 2)
332 | 
333 | 
334 | def example2():
335 |     votelist = [
336 |         ("10", 1000, ["10"]),
337 |         ("20", 1000, ["20"]),
338 |         ("30", 1000, ["30"]),
339 |         ("40", 1000, ["40"]),
340 |         ('2', 500, ['10', '20', '30']),
341 |         ('4', 500, ['10', '20', '40'])
342 |     ]
343 |     run_and_print_all(votelist, 2)
344 | 
345 | 
346 | class MaxScoreTest(unittest.TestCase):
347 |     def test_max_score_1(self):
348 |         votelist = [
349 |             (10, 10.0, [1, 2]),
350 |             (20, 20.0, [1, 3]),
351 |             (30, 30.0, [2, 3]),
352 |         ]
353 |         nomlist, candidates = setuplists(votelist)
354 |         calculate_approval(nomlist)
355 | 
356 |         best, score = calculateMaxScoreNoCutoff(nomlist, candidates)
357 |         self.assertEqual(best.validator_id, 3)
358 |         self.assertEqual(score, 50)
359 | 
360 |     def test_balance_heuristic_example_1(self):
361 |         votelist = [
362 |             (10, 10.0, [1, 2]),
363 |             (20, 20.0, [1, 3]),
364 |             (30, 30.0, [2, 3]),
365 |         ]
366 |         nomlist, winners = phragmms(votelist, 2, 0)
367 |         self.assertEqual(winners[0].validator_id, 3)
368 |         self.assertEqual(winners[1].validator_id, 2)
369 | 
370 |         self.assertEqual(winners[0].backed_stake, 30)
371 |         self.assertEqual(winners[1].backed_stake, 30)
372 | 
373 |     def test_balance_heuristic_example_linear(self):
374 |         votelist = [
375 |             (2, 2000, [11]),
376 |             (4, 1000, [11, 21]),
377 |             (6, 1000, [21, 31]),
378 |             (8, 1000, [31, 41]),
379 |             (110, 1000, [41, 51]),
380 |             (120, 1000, [51, 61]),
381 |             (130, 1000, [61, 71]),
382 |         ]
383 | 
384 |         nomlist, winners = phragmms(votelist, 4, 0)
385 |         self.assertEqual(winners[0].validator_id, 11)
386 |         self.assertEqual(winners[0].backed_stake, 3000)
387 | 
388 |         self.assertEqual(winners[1].validator_id, 31)
389 |         self.assertEqual(winners[1].backed_stake, 2000)
390 | 
391 |         self.assertEqual(winners[2].validator_id, 51)
392 |         self.assertEqual(winners[2].backed_stake, 1500)
393 | 
394 |         self.assertEqual(winners[3].validator_id, 61)
395 |         self.assertEqual(winners[3].backed_stake, 1500)
396 | 
397 | 
398 | 
399 | class ElectionTest(unittest.TestCase):
400 |     def test_phragmen(self):
401 |         votelist = [
402 |             ("A", 10.0, ["X", "Y"]),
403 |             ("B", 20.0, ["X", "Z"]),
404 |             ("C", 30.0, ["Y", "Z"]),
405 |         ]
406 |         nomlist, elected_candidates = seq_phragmen(votelist, 2)
407 |         self.assertEqual(elected_candidates[0].validator_id, "Z")
408 |         self.assertAlmostEqual(elected_candidates[0].score, 0.02)
409 |         self.assertEqual(elected_candidates[1].validator_id, "Y")
410 |         self.assertAlmostEqual(elected_candidates[1].score, 0.04)
411 | 
412 |     def test_approval(self):
413 |         votelist = [
414 |             ("A", 10.0, ["X", "Y"]),
415 |             ("B", 20.0, ["X", "Z"]),
416 |             ("C", 30.0, ["Y", "Z"]),
417 |         ]
418 |         nomlist, elected_candidates = approval_voting(votelist, 2)
419 |         self.assertEqual(elected_candidates[0].validator_id, "Z")
420 |         self.assertAlmostEqual(elected_candidates[0].approval_stake, 50.0)
421 |         self.assertEqual(elected_candidates[1].validator_id, "Y")
422 |         self.assertAlmostEqual(elected_candidates[1].approval_stake, 40.0)
423 | 
424 | 
425 | def main():
426 |     # example1()
427 |     example2()
428 |     # example3()
429 | 


--------------------------------------------------------------------------------
/random/randao_analysis.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "markdown",
  5 |    "metadata": {},
  6 |    "source": [
  7 |     "# Randao Analysis using Markov Chains\n",
  8 |     "\n",
  9 |     "This Sagemath Jupyter notebook is for tha nalaysis of Randao randomness, under the assumption that honest block producers always produce blocks in their slot that always get into the chain and can never be reverted. This assumption is not necessarily reasonable.\n",
 10 |     "\n",
 11 |     "The randomness is sampled at a particular slot, e.g. the end of an epoch or just before it is used by a smart contract. The adversary cannot predict the randomness at the last slot up to this slot that has an honest block producer, because their contribution is random and unknown. However each adversarial blocm producer between this last honest slot and the sampled slot has a choice of whether to produce a block or not. Thus if the adversary controls m slots in a row up to the sampling slot, then they have 2^m choices for the randonness.\n",
 12 |     "\n",
 13 |     "The randomness sampled at the end of epoch n is used to determine the block producers in epoch n+2. We can imagine that the adversary wants to maximise the numnber of adversarial slots at the end of epoch n+2 to get control over the randonness in epoch n+4 etc. If they choose to do this, we can construct a Markov chain, where each state is the number of adversarial blocks at the end of this epoch. The next state is the number of adversarial blocks at the end of the epoch after next. (So odd and even numbered epochs are mostly indepedent.)\n",
 14 |     "\n",
 15 |     "If the adversray controls 1/3 of the validator set and controls m slots at the end of the current epoch then under this attack, the distribution of the number of slots they control at the end of the next epoch is the maximum of 2^m geometric distributions (the kind that start at 0) with parameter 2/3.\n",
 16 |     "\n",
 17 |     "The stationary distribution of this Markov chain is the distribution of the number of adversarial slots at the end of the peoch under continous attack where the adversary tries to maximise this.\n",
 18 |     "\n",
 19 |     "We want to consider sampling randomness 4 epochs after some trigger happens. Now an attacker could wasit until the current epoch has many adversarial validators at the end, before causing the trigger to happen. Then 4 epochs, later, the current epoch may still be somewhate biasable, allowing the adversary to have a more than usual chance to get many adversarial blocks before the trigger block.\n",
 20 |     "\n",
 21 |     "To analyse this, we first need a conservative estimate of how many slots at the end of the current  that adversary can feasibly wait to occur, under the coninuous attack or not. Then we can consider two transitions of the Markov chain from this event as being the distribution of the number of adversarial slots before the randomness is sampled. Now we can compute the expected number of options for the sampled randomness from this distribution."
 22 |    ]
 23 |   },
 24 |   {
 25 |    "cell_type": "code",
 26 |    "execution_count": 1,
 27 |    "metadata": {},
 28 |    "outputs": [],
 29 |    "source": [
 30 |     "# The cumulative distribution function and probability mass functiom\n",
 31 |     "# at m for the maximum of t geometric distributions with parameter p\n",
 32 |     "def cdfmaxgeo(p,t,m):\n",
 33 |     "    return (1-(1-p)^(m+1))^t\n",
 34 |     "def pmfmaxgeo(p,t,m):\n",
 35 |     "    return cdfmaxgeo(p,t,m)-cdfmaxgeo(p,t,m-1)\n",
 36 |     "    "
 37 |    ]
 38 |   },
 39 |   {
 40 |    "cell_type": "code",
 41 |    "execution_count": 2,
 42 |    "metadata": {},
 43 |    "outputs": [],
 44 |    "source": [
 45 |     "# Now we build the transition matrix.\n",
 46 |     "# Each column is the maximum of 2^j geometric distributions with parameter 2/3.\n",
 47 |     "# For the last row, corresponding to m=63, \n",
 48 |     "# we take probability of being at least 63 so the probabilities add up to 1.\n",
 49 |     "def nextstateprob(j,i):\n",
 50 |     "    if i ==63:\n",
 51 |     "        return 1-cdfmaxgeo(2/3+0.0,2^j,62)\n",
 52 |     "    return pmfmaxgeo(2/3+0.0,2^j,i)\n",
 53 |     "tm=matrix([[nextstateprob(j,i) for j in range(64)] for i in range(64)])"
 54 |    ]
 55 |   },
 56 |   {
 57 |    "cell_type": "code",
 58 |    "execution_count": 3,
 59 |    "metadata": {},
 60 |    "outputs": [
 61 |     {
 62 |      "name": "stdout",
 63 |      "output_type": "stream",
 64 |      "text": [
 65 |       "0.999999999952366\n"
 66 |      ]
 67 |     },
 68 |     {
 69 |      "data": {
 70 |       "text/plain": [
 71 |        "(0.476563095838547, 0.290008277069172, 0.139917968194352, 0.0586770548078090, 0.0224471788438172, 0.00810421240432098, 0.00282576354047732, 0.000965620671507453, 0.000326252792938701, 0.000109544499335094, 0.0000366568819313766, 0.0000122441918654699, 4.08585723181877e-6, 1.36273835423470e-6, 4.54384351862505e-7, 1.51485734113910e-7, 5.04995072826173e-8, 1.68339168055658e-8, 5.61143674448838e-9, 1.87050189995125e-9, 6.23504803604958e-10, 2.07835570232040e-10, 6.92786571866131e-11, 2.30930253742276e-11, 7.69755676003532e-12, 2.56585251964193e-12, 8.55525570183441e-13, 2.84933826330784e-13, 9.52193095509939e-14, 3.14984028202337e-14, 1.04994676112577e-14, 3.62050607336612e-15, 1.08615182206194e-15, 3.62050607359248e-16, 3.62050607361851e-16, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000)"
 72 |       ]
 73 |      },
 74 |      "execution_count": 3,
 75 |      "metadata": {},
 76 |      "output_type": "execute_result"
 77 |     }
 78 |    ],
 79 |    "source": [
 80 |     "# Next we use the power method to approximate the stationary distribution stat\n",
 81 |     "pm=tm\n",
 82 |     "for _ in range(20):\n",
 83 |     " pm = pm^2\n",
 84 |     "stat=pm*vector({1:1,63:0})\n",
 85 |     "# The probabilities should add up to 1. \n",
 86 |     "# Too many iterations of the power method will blow up the rounding error, \n",
 87 |     "# so it's better to check that it is not too far from 1.\n",
 88 |     "print(sum(stat))\n",
 89 |     "stat"
 90 |    ]
 91 |   },
 92 |   {
 93 |    "cell_type": "code",
 94 |    "execution_count": 4,
 95 |    "metadata": {},
 96 |    "outputs": [
 97 |     {
 98 |      "data": {
 99 |       "text/plain": [
100 |        "(0.666666666666667, 0.222222222222222, 0.0740740740740741, 0.0246913580246914, 0.00823045267489719, 0.00274348422496573, 0.000914494741655170, 0.000304831580551723, 0.000101610526850648, 0.0000338701756168458, 0.0000112900585389486, 3.76335284635321e-6, 1.25445094878440e-6, 4.18150316261467e-7, 1.39383438679808e-7, 4.64611462636100e-8, 1.54870487545367e-8, 5.16234954783812e-9, 1.72078318261271e-9, 5.73594394204235e-10, 1.91198168408846e-10, 6.37326857955145e-11, 2.12442285985048e-11, 7.08144654026910e-12, 2.36044517265555e-12, 7.86815057551848e-13, 2.62345700718924e-13, 8.73745520379998e-14, 2.91988655476416e-14, 9.65894031423886e-15, 3.21964677141295e-15, 1.11022302462516e-15, 3.33066907387547e-16, 1.11022302462516e-16, 1.11022302462516e-16, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000, 0.000000000000000)"
101 |       ]
102 |      },
103 |      "execution_count": 4,
104 |      "metadata": {},
105 |      "output_type": "execute_result"
106 |     }
107 |    ],
108 |    "source": [
109 |     "# unbiased is the distribution after one transition, which should be just geometric with parameter 2/3\n",
110 |     "unbiased=tm*vector({0:1,63:0})\n",
111 |     "unbiased"
112 |    ]
113 |   },
114 |   {
115 |    "cell_type": "code",
116 |    "execution_count": 5,
117 |    "metadata": {},
118 |    "outputs": [
119 |     {
120 |      "name": "stdout",
121 |      "output_type": "stream",
122 |      "text": [
123 |       "4.64611462636100e-8 4.64611462636100e-8 1.51485734113910e-7\n"
124 |      ]
125 |     }
126 |    ],
127 |    "source": [
128 |     "# Just some sanity checking\n",
129 |     "print(pmfmaxgeo(2/3+0.0,1,15),unbiased[15],stat[15])"
130 |    ]
131 |   },
132 |   {
133 |    "cell_type": "code",
134 |    "execution_count": 6,
135 |    "metadata": {},
136 |    "outputs": [
137 |     {
138 |      "name": "stdout",
139 |      "output_type": "stream",
140 |      "text": [
141 |       "a single transition from ubiased gives expected tail length  0.500000000000001  and expected options  1.99999968601354\n",
142 |       "the steady state distribution has expected tail length  0.904072888428179  and expected options  3.26106196081950\n"
143 |      ]
144 |     }
145 |    ],
146 |    "source": [
147 |     "# Next we look at the expected number of options the adversary has under these distributions\n",
148 |     "def expectedoptions(dist):\n",
149 |     "    return sum([2^i*x for (i,x) in zip(range(64),dist)])\n",
150 |     "def expectedtail(dist):\n",
151 |     "    return sum([i*x for (i,x) in zip(range(64),dist)])\n",
152 |     "print(\"a single transition from ubiased gives expected tail length \",expectedtail(unbiased),\" and expected options \",expectedoptions(unbiased))\n",
153 |     "print(\"the steady state distribution has expected tail length \", expectedtail(stat), \" and expected options \",expectedoptions(stat))"
154 |    ]
155 |   },
156 |   {
157 |    "cell_type": "code",
158 |    "execution_count": 7,
159 |    "metadata": {},
160 |    "outputs": [
161 |     {
162 |      "data": {
163 |       "text/plain": [
164 |        "1.21765601217656e-8"
165 |       ]
166 |      },
167 |      "execution_count": 7,
168 |      "metadata": {},
169 |      "output_type": "execute_result"
170 |     }
171 |    ],
172 |    "source": [
173 |     "# The number of epochs in a century\n",
174 |     "epochsinacentury=(5*60*24*365*1000/32)\n",
175 |     "1/epochsinacentury+0.0"
176 |    ]
177 |   },
178 |   {
179 |    "cell_type": "code",
180 |    "execution_count": 8,
181 |    "metadata": {},
182 |    "outputs": [
183 |     {
184 |      "name": "stdout",
185 |      "output_type": "stream",
186 |      "text": [
187 |       "tail length 15 occurs every 8.03809031457012 years in expectation under the stationary distribution\n",
188 |       "tail length 15 occurs every 26.2080493078638 years in expectation under the single transition from unbiased distribution\n",
189 |       "tail length 16 occurs every 24.1122354988936 years in expectation under the stationary distribution\n",
190 |       "tail length 16 occurs every 78.6241479235913 years in expectation under the single transition from unbiased distribution\n",
191 |       "tail length 17 occurs every 72.3334935202939 years in expectation under the stationary distribution\n",
192 |       "tail length 17 occurs every 235.872445461677 years in expectation under the single transition from unbiased distribution\n"
193 |      ]
194 |     }
195 |    ],
196 |    "source": [
197 |     "#So what tail length can freasibly occur if the adversary waits long enough, \n",
198 |     "# both when they trying to macimise the tail length, which gives the distribution stat\n",
199 |     "# and when they just wait (i.e. using unbiased)\n",
200 |     "print(\"tail length 15 occurs every\",100/(stat[15]*epochsinacentury),\"years in expectation under the stationary distribution\")\n",
201 |     "print(\"tail length 15 occurs every\",100/(unbiased[15]*epochsinacentury),\"years in expectation under the single transition from unbiased distribution\")\n",
202 |     "print(\"tail length 16 occurs every\",100/(stat[16]*epochsinacentury),\"years in expectation under the stationary distribution\")\n",
203 |     "print(\"tail length 16 occurs every\",100/(unbiased[16]*epochsinacentury),\"years in expectation under the single transition from unbiased distribution\")\n",
204 |     "print(\"tail length 17 occurs every\",100/(stat[17]*epochsinacentury),\"years in expectation under the stationary distribution\")\n",
205 |     "print(\"tail length 17 occurs every\",100/(unbiased[17]*epochsinacentury),\"years in expectation under the single transition from unbiased distribution\")"
206 |    ]
207 |   },
208 |   {
209 |    "cell_type": "code",
210 |    "execution_count": 9,
211 |    "metadata": {},
212 |    "outputs": [
213 |     {
214 |      "data": {
215 |       "text/plain": [
216 |        "172.837461872421"
217 |       ]
218 |      },
219 |      "execution_count": 9,
220 |      "metadata": {},
221 |      "output_type": "execute_result"
222 |     }
223 |    ],
224 |    "source": [
225 |     "# If we do two transitions from 16\n",
226 |     "# corresponding to taking a sample 4 epochs after epoch that the adversary timed an atteck for\n",
227 |     "expectedoptions(tm*tm*vector({16:1,63:0}))"
228 |    ]
229 |   },
230 |   {
231 |    "cell_type": "code",
232 |    "execution_count": 10,
233 |    "metadata": {},
234 |    "outputs": [
235 |     {
236 |      "data": {
237 |       "text/plain": [
238 |        "6.41125360736917"
239 |       ]
240 |      },
241 |      "execution_count": 10,
242 |      "metadata": {},
243 |      "output_type": "execute_result"
244 |     }
245 |    ],
246 |    "source": [
247 |     "# and how long is the expected tail then?\n",
248 |     "expectedtail(tm*tm*vector({16:1,63:0}))"
249 |    ]
250 |   },
251 |   {
252 |    "cell_type": "code",
253 |    "execution_count": 11,
254 |    "metadata": {},
255 |    "outputs": [
256 |     {
257 |      "data": {
258 |       "text/plain": [
259 |        "1901.01954391692"
260 |       ]
261 |      },
262 |      "execution_count": 11,
263 |      "metadata": {},
264 |      "output_type": "execute_result"
265 |     }
266 |    ],
267 |    "source": [
268 |     "# What about if we ait 2 epochs instead of 4?\n",
269 |     "expectedoptions(tm*vector({16:1,63:0}))"
270 |    ]
271 |   },
272 |   {
273 |    "cell_type": "code",
274 |    "execution_count": 12,
275 |    "metadata": {
276 |     "scrolled": true
277 |    },
278 |    "outputs": [
279 |     {
280 |      "data": {
281 |       "text/plain": [
282 |        "36.3762992608834"
283 |       ]
284 |      },
285 |      "execution_count": 12,
286 |      "metadata": {},
287 |      "output_type": "execute_result"
288 |     }
289 |    ],
290 |    "source": [
291 |     "# Or 6 epochs?\n",
292 |     "expectedoptions(tm*tm*tm*vector({16:1,63:0}))"
293 |    ]
294 |   },
295 |   {
296 |    "cell_type": "markdown",
297 |    "metadata": {},
298 |    "source": [
299 |     "## What if randomness were sampled at a block number not a slot number?\n",
300 |     "\n",
301 |     "As I know how to query the block number and not the slot number from the EVM, this makes smart contracts getting randomness from RanDAO even trickier. So one might have say entrants to some lottery or the inout to some interactive proof that happens at block n and want to sample RanDAO using the prevRandDAO precompile at block n+128. The problem is that thia need not occur 128 slots later. In slots where the attacker controls the block producer, even when they cannot influence the randomness directky because of a later honest slot, they can still skip producing a block and push the sampling point to a later slot. In this way they can hope to sample in the middle of a strng of adversarial slots.\n",
302 |     "\n",
303 |     "However, there is not much point them doing this until 2 epochs before, because then they don't know which slot to aim for until they know the block producers. Just how many choices of slots do they have, starting at 64 blocks before? If all block producers produce blocks, then the can be a minium of 64 slots. The maximum number of slots before 64 blocks is 64 plus the number of adversarial slots before there are 64 honest slots. If the slots assignments were random, and of course they can be biased as above, this is a sample from a negative binomial distribution with parameters r=64. p=2/3."
304 |    ]
305 |   },
306 |   {
307 |    "cell_type": "code",
308 |    "execution_count": 2,
309 |    "metadata": {},
310 |    "outputs": [],
311 |    "source": [
312 |     "def negbintail(r,p,t):\n",
313 |     "    tail=0.0\n",
314 |     "    for k in range(ceil(t),2*ceil(t)+r):\n",
315 |     "        tail += binomial(k+r-1,k)*p^r*(1-p)^k\n",
316 |     "    return tail"
317 |    ]
318 |   },
319 |   {
320 |    "cell_type": "code",
321 |    "execution_count": 6,
322 |    "metadata": {},
323 |    "outputs": [
324 |     {
325 |      "data": {
326 |       "text/plain": [
327 |        "7774.15709413154"
328 |       ]
329 |      },
330 |      "execution_count": 6,
331 |      "metadata": {},
332 |      "output_type": "execute_result"
333 |     }
334 |    ],
335 |    "source": [
336 |     "# The expectation of this is 33.5, but it takes almost twice that for the tail probability to become small.\n",
337 |     "1/negbintail(67,2/3,64)"
338 |    ]
339 |   },
340 |   {
341 |    "cell_type": "code",
342 |    "execution_count": 12,
343 |    "metadata": {},
344 |    "outputs": [
345 |     {
346 |      "data": {
347 |       "text/plain": [
348 |        "3738.63966703836"
349 |       ]
350 |      },
351 |      "execution_count": 12,
352 |      "metadata": {},
353 |      "output_type": "execute_result"
354 |     }
355 |    ],
356 |    "source": [
357 |     "# If we include the bias from 4 epochs above, the number of options might be 10 more.\n",
358 |     "1/negbintail(67,2/3,74)/172.837461872421"
359 |    ]
360 |   },
361 |   {
362 |    "cell_type": "code",
363 |    "execution_count": null,
364 |    "metadata": {},
365 |    "outputs": [],
366 |    "source": []
367 |   }
368 |  ],
369 |  "metadata": {
370 |   "kernelspec": {
371 |    "display_name": "SageMath 10.1",
372 |    "language": "sage",
373 |    "name": "sagemath"
374 |   },
375 |   "language_info": {
376 |    "codemirror_mode": {
377 |     "name": "ipython",
378 |     "version": 3
379 |    },
380 |    "file_extension": ".py",
381 |    "mimetype": "text/x-python",
382 |    "name": "python",
383 |    "nbconvert_exporter": "python",
384 |    "pygments_lexer": "ipython3",
385 |    "version": "3.11.1"
386 |   }
387 |  },
388 |  "nbformat": 4,
389 |  "nbformat_minor": 4
390 | }
391 | 


--------------------------------------------------------------------------------
/pdf/theory.bib:
--------------------------------------------------------------------------------
  1 | 
  2 | %A
  3 | @article{archetti12game,
  4 |   title={Game theory of public goods in one-shot social dilemmas without assortment},
  5 |   author={Archetti, Marco and Scheuring, Istvan},
  6 |   journal={Journal of theoretical biology},
  7 |   volume={299},
  8 |   pages={9--20},
  9 |   year={2012},
 10 |   publisher={Elsevier}
 11 | } 
 12 | 
 13 | @article{ alistarh18communication,
 14 | 	author = {Dan Alistarh and James Aspnes and Valerie King and
 15 | 			Jared Saia},
 16 | 	title = {\href{https://link.springer.com/article/10.1007/s00446-017-0315-1}{Communication-efficient randomized consensus}},
 17 | 	volume = 31,
 18 | 	pages = {pages489–501},
 19 | 	month = nov,
 20 | 	year = 2018,
 21 | }
 22 | 
 23 | @article{ aspnes03randomized,
 24 | 	author = {James Aspnes},
 25 | 	title = {\href{https://link.springer.com/article/10.1007%2Fs00446-002-0081-5}{Randomized protocols for asynchronous consensus}},
 26 | 	journal = {Distributed Computing},
 27 | 	volume = 16,
 28 | 	number = {2--3},
 29 | 	pages = {165--175},
 30 | 	month = sep,
 31 | 	year = 2003,
 32 | }
 33 | 
 34 | @article{ aspnes15faster,
 35 | 	author = {James Aspnes},
 36 | 	title = {\href{https://link.springer.com/article/10.1007%2Fs00446-013-0195-y}{Faster randomized consensus with an oblivious adversary}},
 37 | 	journal = {Distributed Computing},
 38 | 	volume = 28,
 39 | 	number = 1,
 40 | 	month = feb,
 41 | 	year = 2015,
 42 | 	pages = {21-29},
 43 | }
 44 | 
 45 | @inproceedings{ aumann96efficient,
 46 | 	author = {Yonatan Aumann and Michael A. Bender},
 47 | 	title = {\href{https://doi.org/10.1007/3-540-61440-0_164}{Efficient Asynchronous Consensus with the Value-Oblivious Adversary Scheduler}},
 48 | 	booktitle = {\bibconf[23rd]{ICALP}{International Colloquium on
 49 | 			Automata, Languages and Programming}},
 50 | 	month = jul,
 51 | 	year = 1996,
 52 | 	location = {Paderborn, Germany},
 53 | }
 54 | 
 55 | @article{ aumann05efficient,
 56 | 	author = {Yonatan Aumann and Michael A. Bender},
 57 | 	title = {\href{https://link.springer.com/article/10.1007%2Fs00446-004-0113-4}{Efficient low-contention asynchronous consensus with the value-oblivious adversary scheduler}},
 58 | 	journal = {Distributed Computing},
 59 | 	volume = 17,
 60 | 	number = 3,
 61 | 	month = mar,
 62 | 	year = 2005,
 63 | 	pages = {191-207},
 64 | }
 65 | 
 66 | @article{avarikioti19divide,
 67 |   title={Divide and Scale: Formalization of Distributed Ledger Sharding Protocols},
 68 |   author={Avarikioti, Georgia and Kokoris-Kogias, Eleftherios and Wattenhofer, Roger},
 69 |   journal={arXiv preprint arXiv:1910.10434},
 70 |   year={2019}
 71 | }
 72 | 
 73 | @article{ awerbuch85complexity,
 74 | 	author = {Baruch Awerbuch},
 75 | 	title = {\href{https://dl.acm.org/citation.cfm?id=4227}{Complexity of Network Synchronization}},
 76 | 	journal = {Journal of the Association for Computing Machinery},
 77 | 	volume = 32,
 78 | 	number = 4,
 79 | 	month = oct,
 80 | 	year = 1985,
 81 | 	pages = {804-823},
 82 | }
 83 | 
 84 | 
 85 | %B
 86 | 
 87 | @article{babai2006probability,
 88 |   title={The probability of generating the symmetric group when one of the generators is random},
 89 |   author={Babai, Laszlo and Hayes, Thomas P.},
 90 |   journal={Publ. Math. Debrecen},
 91 |   volume={69},
 92 |   number={3},
 93 |   pages={271--280},
 94 |   year={2006},
 95 |   publisher={Citeseer}
 96 | }
 97 | 
 98 | @inproceedings{ bangalore18almost,
 99 | 	author = {Laasya Bangalore and Ashish Choudhury and Arpita Patra},
100 | 	title = {\href{https://dl.acm.org/citation.cfm?id=3212735}{Almost-Surely Terminating Asynchronous Byzantine Agreement Revisited}},
101 | 	booktitle = {\bibconf{PODC}{Principles of Distributed Computing}},
102 | 	month = jul,
103 | 	year = 2018,
104 | 	pages = {295-304},
105 | }
106 | 
107 | @incollection{bellare03forward,
108 |   title={Forward-security in private-key cryptography},
109 |   author={Bellare, Mihir and Yee, Bennet},
110 |   booktitle = {\bibconf['03]{CT-RSA}{Topics in Cryptology - CT RSA  }},
111 |   year={2003},
112 | }
113 | 
114 | @inproceedings{bellare93defining,
115 |   title={On defining proofs of knowledge},
116 |   author={Bellare, Mihir and Goldreich, Oded},
117 |   booktitle = {\bibconf['92]{CRYPTO}{Advances in Cryptology  }},
118 |   year={1993},
119 | }
120 | 
121 | @inproceedings{ ben-or83another,
122 | 	author = {Michael Ben-Or},
123 | 	title = {Another advantage of free choice:
124 | 			Completely asynchronous agreement protocols},
125 | 	booktitle = {Principles of Distributed Computing (PODC)},
126 | 	year = 1983,
127 | }
128 | 
129 | @inproceedings{ ben-or85fast,
130 | 	author = {Michael Ben-Or},
131 | 	title = {\href{https://dl.acm.org/citation.cfm?id=323609}{Fast Asynchronous Byzantine Agreement (Extended Abstract)}},
132 | 	booktitle = {\bibconf[4th]{PODC}{Principles of Distributed Computing}},
133 | 	year = 1985,
134 | 	pages = {149-151},
135 | 	location = {Minaki, Ontario, Canada},
136 | }
137 | 
138 | @inproceedings{ bracha84asynchronous,
139 | 	author = {Gabriel Bracha},
140 | 	title = {\href{https://dl.acm.org/citation.cfm?id=806743}{An asynchronous [(n-1)/3]-Resilient Consensus Protocol}},
141 | 	booktitle = {\bibconf[3rd]{PODC}{ACM Symposium on Principles of Distributed Computing}},
142 | 	year = 1984,
143 | 	location = {Vancouver, British Columbia, Canada},
144 | }
145 | % pages = {154-162},
146 | 
147 | @article{ bracha85asynchronous,
148 | 	author = {Gabriel Bracha and Sam Toueg},
149 | 	title = {\href{https://dl.acm.org/citation.cfm?id=214134}{Asynchronous Consensus and Broadcast Protocols}},
150 | 	journal = {Journal of the Association for Computing Machinery (JACM)},
151 | 	volume = 32,
152 | 	number = 4,
153 | 	year = 1985,
154 | }
155 | % pages = {824-840},
156 | 
157 | 
158 | %C
159 | 
160 | @book{ cachin11introduction,
161 | 	author = {Christian Cachin and Rachid Guerraoui Lu\'is Rodrigues},
162 | 	title  = {Introduction to Reliable and Secure Distributed Programming},
163 | 	publisher = {Springer},
164 | 	month = feb,
165 | 	year = 2011,
166 | 	isbn = {978-3642152597},
167 | }
168 | 
169 | @misc{ cachin19asymmetric,
170 | 	author = {Christian Cachin and Bj\"orn Tackmann},
171 | 	title = {\href{https://arxiv.org/pdf/1906.09314.pdf}{Asymmetric Distributed Trust}},
172 | 	month = jun,
173 | 	year = 2019,
174 | }
175 | 
176 | @techreport{ camenisch97proof,
177 |   author = {Jan Camenisch and Markus Stadler},
178 |   title = {Proof Systems for General Statements about Discrete
179 |       Logarithms},
180 |   institution = {Dept. of Computer Science, ETH Zurich},
181 |   year = 1997,
182 |   month = {March},
183 |   number = 260,
184 | }
185 | 
186 | @inproceedings{ canetti93fast,
187 | 	author = {Ran Canetti and Tal Rabin},
188 | 	title = {\href{https://dl.acm.org/citation.cfm?id=167105}{Fast Asynchronous Byzantine Agreement with Optimal Resilience}},
189 | 	booktitle = {\bibconf[25th]{STOC}{ACM Symposium on Theory of computing}},
190 | 	month = may,
191 | 	year = 1993,
192 | 	location = {San Diego, California, USA},
193 | 	pages = {42-51},
194 | }
195 | 
196 | % long version: unpublished?
197 | @misc{ canetti98fast,
198 | 	author = {Ran Canetti and Tal Rabin},
199 | 	title = {\href{http://people.csail.mit.edu/canetti/materials/cr93.ps}{Fast Asynchronous Byzantine Agreement with Optimal Resilience}},
200 | 	month = sep,
201 | 	year = 1998,
202 | 	institution = {IBM T.J. Watson Research Center},
203 | }
204 | 
205 | @inproceedings{ clarkson08civitas ,
206 | 	author = {Clarkson, M.R. and Chong, S. and Myers, A.C.},
207 | 	booktitle = {IEEE \bibconf{SP}{Symposium on Security and Privacy}},
208 | 	title = {Civitas: Toward a Secure Voting System},
209 | 	year = {2008},
210 | 	month = {may},
211 | }
212 | 
213 | @article{ cristian95atomic,
214 | 	author = {Flaviu Cristian and Houtan Aghili and Ray Strong and
215 | 			Danny Dolev},
216 | 	title = {\href{https://www.sciencedirect.com/science/article/pii/S0890540185710607}{Atomic Broadcast: From Simple Message Diffusion to Byzantine Agreement}},
217 | 	journal = {Information and Computation},
218 | 	volume = 118,
219 | 	number = 1,
220 | 	month = apr,
221 | 	year = 1995,
222 | 	pages = {158-179},
223 | }
224 | 
225 | %D
226 | 
227 | @article{ defago04total,
228 | 	author = {Xavier D\'efago and Andr\'e Schiper and P\'eter Urb\'an},
229 | 	title = {\href{https://dl.acm.org/doi/abs/10.1145/1041680.1041682}{Total Order Broadcast and Multicast Algorithms:Taxonomy and Survey}},
230 | 	journal = {ACM Computing Surveys},
231 | 	month = dec,
232 | 	year = 2004,
233 | }
234 | 
235 | %F
236 | 
237 | @article{ feigenbaum08graph,
238 | 	author = {Joan Feigenbaum and Sampath Kannan and Andrew McGregor
239 | 			and Siddharth Suri and Jian Zhang},
240 | 	title = {Graph Distances in the Data-Stream Model},
241 | 	journal = {SIAM Journal on Computing},
242 | 	volume = 38,
243 | 	year = 2008,
244 | 	pages = {1709-1727},
245 | }
246 | 
247 | @article{ feigenbaum05graph,
248 | 	author = {Joan Feigenbaum and Sampath Kannan and Andrew McGregor
249 | 			and Siddharth Suri and Jian Zhang},
250 | 	title = {On Graph Problems in a Semi-Streaming Model},
251 | 	journal = {Theoretical Computer Science},
252 | 	volume = 348,
253 | 	year = 2005,
254 | 	pages = {207-216},
255 | }
256 | 
257 | @article{ feigenbaum05computing,
258 | 	author = {Joan Feigenbaum and Sampath Kannan and Jian Zhang},
259 | 	title = {Computing Diameter in the Streaming and Sliding-Window Models},
260 | 	journal = {Algorithmica},
261 | 	volume = 41,
262 | 	year = 2005,
263 | 	pages = {25-41},
264 | }
265 | 
266 | @inproceedings{ feldman88optimal,
267 | 	author = {Paul Feldman and Silvio Micali},
268 | 	title = {\href{https://dl.acm.org/citation.cfm?id=62225}{Optimal Algorithms for Byzantine Agreement}},
269 | 	booktitle = {\bibconf[20th]{STOC}{Symposium on Theory of Computing}},
270 | 	month = may,
271 | 	year = 1988,
272 | 	pages = {148-161},
273 | 	location = {Chicago, Illinois, USA},
274 | }
275 | 
276 | @phdthesis{ feldman88thesis,
277 | 	author = {Paul Feldman},
278 | 	title = {\href{https://dspace.mit.edu/bitstream/handle/1721.1/14368/20051076-MIT.pdf}{Optimal Algorithms for Byzantine Agreement}},
279 | 	school = {Massachusetts Institute of Technology},
280 | 	month = may,
281 | 	year = 1988,
282 | }
283 | 
284 | @article{ fischer85impossibility,
285 |   title={\href{https://groups.csail.mit.edu/tds/papers/Lynch/jacm85.pdf}{Impossibility of distributed consensus with one faulty process}},
286 |   author={Fischer, Michael J and Lynch, Nancy A and Paterson, Michael S},
287 |   journal={Journal of the ACM (JACM)},
288 |   volume={32},
289 |   number={2},
290 |   pages={374--382},
291 |   year={1985},
292 |   publisher={ACM}
293 | }
294 | 
295 | @article{ friedman05simple,
296 | 	author = {Roy Friedman and Achour Mostefaoui and Michel Raynal},
297 | 	title = {Simple and Efficient Oracle-Based Consensus Protocols
298 | 			for Asynchronous {Byzantine} Systems},
299 | 	journal = {IEEE Transactions on Dependable and Secure Computing},
300 | 	volume = 2,
301 | 	number = 1,
302 | 	month = jan,
303 | 	year = 2005,
304 | }
305 | 
306 | 
307 | %J
308 | 
309 | @book{ johnson77urn,
310 | 	author = {Norman Lloyd Johnson},
311 | 	title = {Urn models and their application: An approach to modern discrete probability theory},
312 | 	publisher = {Wiley},
313 | 	year = 1977,
314 | 	isbn = {978-0471446309},
315 | }
316 | 
317 | %K
318 | 
319 | %L
320 | 
321 | @book{ lynch96distributed,
322 | 	author = {Nancy A. Lynch},
323 | 	title = {Distributed Algorithms},
324 | 	publisher = {Morgan Kaufmann},
325 | 	month = mar,
326 | 	year = 1996,
327 | 	isbn = {978-1558603486},
328 | }
329 | 
330 | 
331 | %M
332 | 
333 | @book{ mahmoud08polya,
334 | 	author = {Hosam Mahmoud},
335 | 	title = {P\'olya Urn Models},
336 | 	publisher = {Chapman and Hall/CRC},
337 | 	month = jun,
338 | 	year = 2008,
339 | 	isbn = {978-1420059830},
340 | }
341 | 
342 | @inproceedings{merkle88digital,
343 |   title={\href{https://people.eecs.berkeley.edu/~raluca/cs261-f15/readings/merkle.pdf}{A Digital Signature Based on a Conventional Encryption Function}},
344 |   author={Merkle, Ralph C},
345 |   booktitle={\bibconf{CRYPTO}{Advances in Cryptology}},
346 |   year={1988},
347 | }
348 | 
349 | @article{ moran58random,
350 | 	author = {{P. A. P.} Moran},
351 | 	title = {Random Processes in Genetics},
352 | 	journal = {\href{https://doi.org/10.1017/S0305004100033193}{Mathematical Proceedings of the Cambridge Philosophical Society}},
353 | 	volume = 54,
354 | 	number = 1,
355 | 	month = jan,
356 | 	year = 1958,
357 | 	pages = {60-71},
358 | }
359 | 
360 | @inproceedings{ mostefaoui14signature,
361 | 	author = {Achour Most\'efaoui and Hamouma Moumen and Michel Raynal},
362 | 	title = {\href{https://dl.acm.org/citation.cfm?id=2611468}{Signature-Free Asynchronous Byzantine Consensus with $t < n/3$ and $O(n^2)$ Messages}},
363 | 	booktitle = {\bibconf{PODC}{Principles of Distributed Computing}},
364 | 	month = jul,
365 | 	year = 2014,
366 | 	location = {Paris, France},
367 | }
368 | 
369 | @inproceedings{munro92detskiplists,
370 |  author = {Munro, J. Ian and Papadakis, Thomas and Sedgewick, Robert},
371 |  title = {\href{http://www.ic.unicamp.br/~celio/peer2peer/skip-net-graph/deterministic-skip-lists-munro.pdf}{Deterministic Skip Lists}},
372 |  booktitle = {Proceedings of the Third Annual ACM-SIAM Symposium on Discrete Algorithms},
373 |  series = {SODA '92},
374 |  year = {1992},
375 |  isbn = {0-89791-466-X},
376 |  location = {Orlando, Florida, USA},
377 |  pages = {367--375},
378 |  numpages = {9},
379 |  url = {http://dl.acm.org/citation.cfm?id=139404.139478},
380 |  acmid = {139478},
381 |  publisher = {Society for Industrial and Applied Mathematics},
382 |  address = {Philadelphia, PA, USA},
383 | } 
384 | 
385 | 
386 | %N
387 | 
388 | @book{ nowak06evolutionary,
389 | 	author = {Martin A. Nowak},
390 | 	title = {Evolutionary Dynamics: Exploring the Equations of Life},
391 | 	publisher = {Belknap Press},
392 | 	month = sep,
393 | 	year = 2006,
394 | 	isbn = {978-0674023383},
395 | }
396 | 
397 | 
398 | %P
399 | 
400 | @article{pease80reaching,
401 | 	author={Pease, Marshall and Shostak, Robert and Lamport, Leslie},
402 | 	title={\href{http://dl.acm.org/citation.cfm?id=322188}{Reaching Agreement in the Presence of Faults}},
403 | 	journal={Journal of the ACM (JACM)},
404 | 	volume={27},
405 | 	number={2},
406 | 	pages={228--234},
407 | 	month=apr,
408 | 	year={1980},
409 | 	publisher={ACM}
410 | }
411 | 
412 | @misc{ propp15polyas,
413 | 	author = {James Propp},
414 | 	title = {\href{https://mathenchant.wordpress.com/2015/10/16/polyas-urn/}{P\'olya's Urn}},
415 | 	month = oct,
416 | 	year = 2015,
417 | }
418 | 
419 | @article{pugh90skiplists,
420 |  author = {Pugh, William},
421 |  title = 
422 |  {\href{http://courses.cs.vt.edu/cs2604/fall05/wmcquain/Notes/Supplemental/PughSkiplistPaper.pdf}{Skip
423 |   Lists: A Probabilistic Alternative to Balanced Trees}},
424 |  journal = {Communications of the ACM},
425 |  issue_date = {June 1990},
426 |  volume = {33},
427 |  number = {6},
428 |  month = jun,
429 |  year = {1990},
430 |  issn = {0001-0782},
431 |  pages = {668--676},
432 |  numpages = {9},
433 |  url = {http://doi.acm.org/10.1145/78973.78977},
434 |  doi = {10.1145/78973.78977},
435 |  acmid = {78977},
436 |  publisher = {ACM},
437 |  address = {New York, NY, USA},
438 |  keywords = {data structures, searching, trees},
439 | } 
440 | 
441 | 
442 | %R
443 | 
444 | @inproceedings{ rabin83randomized,
445 | 	author = {Michael O. Rabin},
446 | 	title = {Randomized {Byzantine} Generals},
447 | 	booktitle = {Symposium on Foundations of Computer Science (SFCS)},
448 | 	month = nov,
449 | 	year = 1983,
450 | }
451 | 
452 | @book{rapoport65prisoner,
453 |   title={Prisoner's dilemma: A study in conflict and cooperation},
454 |   author={Rapoport, Anatol and Chammah, Albert M and Orwant, Carol J},
455 |   volume={165},
456 |   year={1965},
457 |   publisher={University of Michigan press}
458 | }
459 | 
460 | %S
461 | 
462 | @article{skala13hypergeometric,
463 |   author    = {Matthew Skala},
464 |   title     = {{Hypergeometric Tail Inequalities: Ending the Insanity}},
465 |   journal   = {CoRR},
466 |   volume    = {abs/1311.5939},
467 |   year      = {2013},
468 |   url       = {https://arxiv.org/abs/1311.5939},
469 | }
470 | 
471 | @inproceedings{ stadler96publicly,
472 |   title={\href{https://link.springer.com/content/pdf/10.1007/3-540-68339-9_17.pdf}{Publicly Verifiable Secret Sharing}},
473 |   author={Stadler, Markus},
474 |   booktitle = {Eurocrypt},
475 |   month=may,
476 |   year={1996},
477 | }
478 | 
479 | @inproceedings{
480 | CJI09,  author = "R.~Chang and G.~Jiang and F.~Ivan{\v{c}}i{\'{c}} and
481 | 	 S.~Sankaranarayanan and V.~Shmatikov",
482 | title = "Inputs of Coma:
483 | 	 Static Detection of Denial-of-Serice Vulnerabilities",
484 | year = {2009},
485 | booktitle ={CSF}}
486 | 
487 | @article(
488 | CMS05, author="R.~Chadha and J.C.~Mitchell and A.~Scedrov and V.~Shmatikov",
489 | title="Contract signing, optimism, and advantage",
490 | journal="J. Logic and Algebraic Programming",
491 | pages="189--218",
492 | volume="64", number="2",
493 | Year="2003")
494 | 
495 | @inproceedings(
496 | DS,   Author="A.~Serjantov and G.~Danezis",
497 | Title="Towards an information theoretic metric for anonymity",
498 | Booktitle="Proc.\ 2nd International Workshop on
499 | 	Privacy-Enhancing Technologies",
500 | Series={LNCS},
501 | Volume={2482},
502 | Pages="41--53",
503 | Year="2002")
504 | 
505 | @inproceedings(
506 | MS05,   author="A.~Mahimkar and V.~Shmatikov",
507 | title="Game-based analysis of denial-of-service prevention protocols",
508 | booktitle="Proc.\ 18th {IEEE} Computer Security Foundations
509 | 	  Workshop ({CSFW})",
510 | publisher="{IEEE}",
511 | year="2005",
512 | pages="287--301")
513 | 
514 | @article(
515 | NS06,   Author="G.~Norman and V.~Shmatikov",
516 | Title="Analysis of probabilistic contract signing",
517 | Journal="J. Computer Security",
518 | Year="2006",
519 | Pages="561--589",
520 | Volume="14",
521 | Number="6")
522 | 
523 | @inproceedings(
524 | nymble, author="P.~Johnson and A.~Kapadia and P.~Tsang and S.~Smith",
525 | title="Nymble: anonymous {IP}-address blocking",
526 | booktitle="Proc.\ {PET}",
527 | year="2007")
528 | 
529 | @article(
530 | S04,    Author="V.~Shmatikov",
531 | Title="Probabilistic model checking of an anonymity system",
532 | Journal="J. Computer Security",
533 | Year="2004",
534 | Pages="355--377",
535 | Volume="12",
536 | Number="3-4")
537 | 
538 | @article(
539 | SM02,   author="V.~Shmatikov and J.C.~Mitchell",
540 | title="Finite-state analysis of two contract signing protocols",
541 | journal="Theoretical Computer Science",
542 | volume=283, number="2",
543 | pages="419-450",
544 | year=2002)
545 | 
546 | @inproceedings(
547 | SW-esorics06, author="V.~Shmatikov and M-H.~Wang",
548 | Title="Timing analysis in low-latency mix networks:
549 |       attacks and defenses",
550 | Booktitle="Proc.\ {ESORICS}",
551 | Year="2006")
552 | 
553 | @inproceedings(
554 | SW-fmse06, author="V.~Shmatikov and M-H.~Wang",
555 | Title="Measuring relationship anonymity in mix networks",
556 | Booktitle="Proc.\ {WPES}",
557 | Year="2006")
558 | 
559 | @inproceedings(
560 | torsk-attack, author="Q.~Wang and P.~Mittal and N.~Borisov",
561 | title="In Search of an Anonymous and Secure Lookup:
562 |        Attacks on Peer-to-peer Anonymous Communication Systems",
563 | booktitle="Proc.\ {CCS}",
564 | year="2010")
565 | 
566 | @inproceedings( tsang-ccs07,
567 | 	author="P.~Tsang and M.H.~Au and A.~Kapadia and S.~Smith",
568 | 	title="Blacklistable anonymous credentials:
569 | 	      blocking misbehaving users without {TTPs}",
570 | 	booktitle="Proc.\ {CCS}",
571 | 	year="2007"
572 | )
573 | 
574 | @inproceedings(
575 | tsang-ccs08, author="P.~Tsang and M.H.~Au and A.~Kapadia and S.~Smith",
576 | title="{PEREA}: Towards Practical {TTP}-Free Revocation in
577 |       Anonymous Authentication",
578 | booktitle="Proc.\ {CCS}",
579 | year="2008")
580 | 
581 | 
582 | @article{ afk,
583 | 	author = {Mart\'in Abadi and Joan Feigenbaum and Joe Kilian},
584 | 	title = {On Hiding Information from an Oracle},
585 | 	journal = {Journal of Computer and System Sciences},
586 | 	volume = 39,
587 | 	year = 1989,
588 | 	pages = {21-50},
589 | 	note = {Special issue of selected papers from the 1987
590 | 		IEEE Conference on Structure in Complexity Theory}
591 | },
592 | 
593 | @inproceedings{ bfl,
594 | 	author = {Matt Blaze and Joan Feigenbaum and Jack Lacy},
595 | 	title = {Decentralized Trust Management},
596 | 	booktitle = {Proceedings of the 17th Symposium on Security and Privacy},
597 | 	year = 1996,
598 | 	pages = {164-173},
599 | }
600 | 
601 | @inproceedings{ bfs,
602 | 	author = {Matt Blaze and Joan Feigenbaum and Martin Strauss},
603 | 	title = {Compliance Checking in the {PolicyMaker}
604 | 			Trust Management System},
605 | 	booktitle = {Proceedings of the 2nd Financial Crypto Conference},
606 | 	year = 1998,
607 | }
608 | 
609 | @article{ ff,
610 | 	author = {Joan Feigenbaum and Lance Fortnow},
611 | 	title = {Random-Self-Reducibility of Complete Sets},
612 | 	journal = {SIAM Journal on Computing},
613 | 	volume = 22,
614 | 	year = 1993,
615 | 	pages = {994-1005},
616 | 	note = {Extended abstract appears in Proceedings of the
617 | 		1991 IEEE Conference on Structure in Complexity Theory},
618 | }
619 | 
620 | @article{ fksv,
621 | 	author = {Joan Feigenbaum and Sampath Kannan and Martin Strauss
622 | 		and Mahesh Viswanathan},
623 | 	title = {An Approximate {L1}-Difference Algorithm for
624 | 		Massive Data Streams},
625 | 	journal = {SIAM Journal on Computing},
626 | 	volume = 32,
627 | 	year = 2002,
628 | 	pages = {131--151},
629 | 	note = {Extended abstract appears in Proceedings of the
630 | 		1999 IEEE Symposium on Foundations of Computer Science},
631 | }
632 | 
633 | @article{ fkmsz,
634 | 	author = {Joan Feigenbaum and Sampath Kannan and Andrew McGregor
635 | 			and Siddharth Suri and Jian  Zhang},
636 | 	title = {On Graph Problems in a Semi-Streaming Model},
637 | 	journal = {Theoretical Computer Science},
638 | 	volume = 348,
639 | 	year = 2005,
640 | 	pages = {207--216},
641 | 	note = {Special issue of selected papers from the 2004 International
642 | 		Colloquium on Automata, Languages, and Programming},
643 | }
644 | 
645 | @article{ fps,
646 | 	author = {Joan Feigenbaum and Christos Papadimitriou and Scott Shenker},
647 | 	title = {Sharing the Cost of Multicast Transmissions},
648 | 	journal = {Journal of Computer and System Sciences},
649 | 	volume = 63,
650 | 	year = 2001, 
651 | 	pages = {21--41},
652 | 	note = {Preliminary version appears in Proceedings of the 2000
653 | 		ACM Symposium on Theory of Computing},
654 | }
655 | 
656 | @article{ fpss,
657 | 	author = { Joan Feigenbaum and Christos Papadimitriou and Rahul Sami
658 | 			and Scott Shenker},
659 | 	title = {A {BGP}-based Mechanism for Lowest-Cost Routing},
660 | 	journal = {Distributed Computing},
661 | 	volume = 18,
662 | 	year = 2005,
663 | 	pages = {61--72},
664 | 	note = {Special issue of selected papers from the 2002 ACM Symposium
665 | 		on Principles of Distributed Computing},
666 | }
667 | 
668 | @inproceedings{shoup00practical,
669 |   title={\href{https://link.springer.com/content/pdf/10.1007/3-540-45539-6_15.pdf}{Practical Threshold Signatures}},
670 |   author={Shoup, Victor},
671 |   booktitle={Eurocrypt},
672 |   month = may,
673 |   year={2000},
674 | }
675 | 
676 | 
677 | @book{stinson05crypto,
678 |   author = {Douglas R. Stinson},
679 |   title = {Cryptography: Theory and Practice},
680 |   year = {2005},
681 | }
682 | 
683 | @inproceedings{guerraoui10next,
684 |   title={\href{http://www.vukolic.com/700-Eurosys.pdf}{The next 700 {BFT} protocols}},
685 |   author={Guerraoui, Rachid and Kne{\v{z}}evi{\'c}, Nikola and Qu{\'e}ma, Vivien and Vukoli{\'c}, Marko},
686 |   booktitle={5th European conference on Computer systems},
687 |   pages={363--376},
688 |   year={2010},
689 |   organization={ACM},
690 |   url={http://www.vukolic.com/700-Eurosys.pdf},
691 | }
692 | 
693 | 
694 | %V
695 | 
696 | @book{ vanderbei13linear,
697 | 	author = {Robert J. Vanderbei},
698 | 	title = {Linear Programming: Foundations and Extensions},
699 | 	publisher = {Springer},
700 | 	month = jul,
701 | 	year = 2013,
702 | 	isbn = {978-1461476290},
703 | }
704 | 
705 | 


--------------------------------------------------------------------------------
/NPoS/ComplicatedPhragmén.py:
--------------------------------------------------------------------------------
  1 | #from itertools import count
  2 | import math
  3 | class edge:
  4 |     def __init__(self,voterid,canid):
  5 |         self.voterid=voterid
  6 |         self.canid=canid
  7 |         #self.index
  8 |         #self.voterindex
  9 |         #self.canindex
 10 | 
 11 | 
 12 | class voter:
 13 |     def __init__(self,votetuple):
 14 |         self.voterid=votetuple[0]
 15 |         self.budget=votetuple[1]
 16 |         self.edges=[edge(self.voterid,canid) for canid in votetuple[2]]
 17 |         #self.index
 18 | 
 19 | class candidate:
 20 |     def __init__(self,canid,index):
 21 |         self.canid = canid
 22 |         self.index=index
 23 | 
 24 | 
 25 | import itertools
 26 | class assignment:
 27 |     def __init__(self,voterlist,candidates, copyassignment=None):
 28 |         self.voterlist=voterlist
 29 |         self.candidates=candidates
 30 |         if copyassignment is None:
 31 |             #create edgelist here at cost O(votes size)
 32 |             self.edgelist = list(itertools.chain.from_iterable((nom.edges for nom in voterlist)))
 33 |             numvoters = len(voterlist)
 34 |             numcandidates = len(candidates)
 35 |             numedges=len(self.edgelist)
 36 |             self.voterload=[0.0 for x in range(numvoters)]
 37 |             self.edgeload = [0.0 for x in range(numedges)]
 38 |             self.edgeweight = [0.0 for x in range(numedges)]
 39 |             self.cansupport=[0.0 for x in range(numcandidates)]
 40 |             self.canelected=[False for x in range(numcandidates)]
 41 |             self.electedcandidates=set()
 42 |             self.canapproval= [0.0 for x in range(numcandidates)]
 43 |             #calculate approval here at cost O(numedges)
 44 |             for voter in voterlist:
 45 |                 for edge in voter.edges:
 46 |                     self.canapproval[edge.canindex] += voter.budget
 47 |             self.canscore = [0.0 for x in range(numcandidates)]
 48 |             self.canscorenumerator = [0.0 for x in range(numcandidates)]
 49 |             self.canscoredenominator = [0.0 for x in range(numcandidates)]
 50 |         else:
 51 |             self.edgelist = copyassignment.edgelist
 52 |             self.voterload=copyassignment.voterload.copy()
 53 |             self.edgeload = copyassignment.edgeload.copy()
 54 |             self.edgeweight=copyassignment.edgeweight.copy()
 55 |             self.cansupport=copyassignment.cansupport.copy()
 56 |             self.canelected=copyassignment.canelected.copy()
 57 |             self.electedcandidates=copyassignment.electedcandidates.copy()
 58 |             self.canapproval=copyassignment.canapproval.copy()
 59 |             self.canscore=copyassignment.canscore.copy()
 60 |             self.canscorenumerator = copyassignment.canscorenumerator.copy()
 61 |             self.canscoredenominator = copyassignment.canscoredenominator.copy()
 62 |     def setload(self, edge,load):
 63 |         oldload=self.edgeload[edge.index]
 64 |         self.edgeload[edge.index]=load
 65 |         self.voterload[edge.voterindex] +=load-oldload
 66 |     def setweight(self,edge,weight):
 67 |         oldweight=self.edgeweight[edge.index]
 68 |         self.edgeweight[edge.index]=weight
 69 |         self.cansupport[edge.canindex] +=weight-oldweight
 70 |     def setscore(self,candidate,score):
 71 |         self.canscore[candidate.index] = score
 72 |     def loadstoweights(self):
 73 |         for voter in self.voterlist:
 74 |             for edge in voter.edges:
 75 |                 if(self.voterload[voter.index] > 0.0):
 76 |                     self.setweight(edge, voter.budget * self.edgeload[edge.index] / self.voterload[voter.index])
 77 |     def weightstoloads(self):
 78 |         for edge in self.edgelist:
 79 |             self.setload(edge, self.edgeweight[edge.index]/self.cansupport[edge.canindex])
 80 |     def elect(self,candidate):
 81 |         self.canelected[candidate.index]=True
 82 |         self.electedcandidates.add(candidate)
 83 |     def unelect(self,candidate):
 84 |         self.canelected[candidate.index]=False
 85 |         self.electedcandidates.remove(candidate)
 86 | 
 87 | def setuplists(votelist):
 88 |     #Instead of Python's dict here, you can use anything with O(log n) addition and lookup.
 89 |     #We can also use a hashmap, by generating a random constant r and useing H(canid+r)
 90 |     #since the naive thing is obviously attackable.
 91 |     voterlist = [voter(votetuple) for votetuple in votelist]
 92 |     candidatedict=dict()
 93 |     candidatearray=list()
 94 |     numcandidates=0
 95 |     numvoters=0
 96 |     numedges=0
 97 | 
 98 |     #Get an array of candidates that we can reference these by index
 99 |     for nom in voterlist:
100 |         nom.index=numvoters
101 |         numvoters+= 1
102 |         for edge in nom.edges:
103 |             edge.index=numedges
104 |             edge.voterindex=nom.index
105 |             numedges += 1
106 |             canid = edge.canid
107 |             if canid in candidatedict:
108 |                 edge.candidate=candidatearray[candidatedict[canid]]
109 |                 edge.canindex=edge.candidate.index
110 |             else:
111 |                 candidatedict[canid]=numcandidates
112 |                 newcandidate=candidate(canid,numcandidates)
113 |                 candidatearray.append(newcandidate)
114 |                 edge.candidate=newcandidate
115 |                 edge.canindex=numcandidates
116 |                 numcandidates += 1
117 |     return(voterlist,candidatearray)
118 | 
119 | 
120 | def seqPhragmén(votelist,numtoelect):
121 |     nomlist,candidates=setuplists(votelist)
122 |     #creating an assignment now also computes the total possible stake for each candidate
123 |     a=assignment(nomlist,candidates)
124 | 
125 |     for round in range(numtoelect):
126 |         for canindex in range(len(candidates)):
127 |             if not a.canelected[canindex]:
128 |                 a.canscore[canindex]=1/a.canapproval[canindex]
129 |         for nom in a.voterlist:
130 |             for edge in nom.edges:
131 |                 if not a.canelected[edge.canindex]:
132 |                     a.canscore[edge.canindex] += nom.budget * a.voterload[nom.index] / a.canapproval[edge.canindex]
133 |         bestcandidate=0
134 |         bestscore = 1000 #should be infinite but I'm lazy
135 |         for canindex in range(len(candidates)):
136 |             if not a.canelected[canindex] and a.canscore[canindex] < bestscore:
137 |                 bestscore=a.canscore[canindex]
138 |                 bestcandidate=canindex
139 |         electedcandidate=candidates[bestcandidate]
140 |         a.canelected[bestcandidate]=True
141 |         #electedcandidate.electedpos=round
142 |         a.elect(electedcandidate)
143 |         for nom in a.voterlist:
144 |             for edge in nom.edges:
145 |                 if edge.canindex == bestcandidate:
146 |                     a.setload(edge,a.canscore[bestcandidate]-a.voterload[nom.index])
147 |     a.loadstoweights()
148 |     return a
149 | 
150 | def calculateScores(a,cutoff):
151 |     for canindex in range(len(a.candidates)):
152 |         if not a.canelected[canindex]:
153 |             a.canscorenumerator[canindex]=0
154 |             a.canscoredenominator[canindex]=1
155 |     for nom in a.voterlist:
156 |         numeratorcontrib=nom.budget
157 |         denominatorcontrib=0
158 |         for edge in nom.edges:
159 |             if a.canelected[edge.canindex]:
160 |                 if a.cansupport[edge.canindex] > cutoff:
161 |                     denominatorcontrib += a.edgeweight[edge.index]/a.cansupport[edge.canindex]
162 |                 else:
163 |                     numeratorcontrib -= a.edgeweight[edge.index]
164 |         for edge in nom.edges:
165 |             if not a.canelected[edge.canindex]:
166 |                 a.canscorenumerator[edge.canindex] +=numeratorcontrib
167 |                 a.canscoredenominator[edge.canindex] +=denominatorcontrib
168 |     for canindex in range(len(a.candidates)):
169 |         if not a.canelected[canindex]:
170 |            a.canscore[canindex] = a.canscorenumerator[canindex]/a.canscoredenominator[canindex]
171 |     #for canindex in range(len(a.candidates)):
172 |         #if not a.canelected[canindex]:
173 |             #print(a.candidates[canindex].canid," has score ", a.canscore[canindex]," with cutoff ",cutoff)
174 |             #print("Approval stake: ", a.canapproval[canindex]," support: ",a.cansupport[canindex]," denominator: ",a.canscoredenominator[canindex], " numerator: ",a.canscorenumerator[canindex])
175 | 
176 | 
177 | def calculateMaxScore(a):
178 |     supportList=[a.cansupport[can.index] for can in a.electedcandidates]
179 |     supportList.append(0.0)
180 |     supportList.sort()
181 |     lowerindex=0
182 |     upperindex=len(a.electedcandidates)+1
183 |     currentindex=0
184 |     while(True):
185 |         #print(len(supportList), currentindex, len(a.electedcandidates),upperindex,lowerindex)
186 |         cutoff=supportList[currentindex]
187 |         calculateScores(a,cutoff)
188 |         scores=[(can, a.canscore[can.index]) for can in a.candidates if not a.canelected[can.index]]
189 |         bestcandidate,score=max(scores,key=lambda x: x[1])
190 |         if score > cutoff:
191 |             # In this case both score and cutoff are lower bounds to the max score
192 |             lowerindex=len([s for s in supportList if s <= score]) - 1
193 |             #print("lowerindex ",lowerindex, " upperindex ",upperindex, " cutoff ", cutoff, "score ",score, " candidate ",bestcandidate.canid)
194 |             #print("bottom support ",supportList[0], " real lowest support ",supportList[1], " highest support ",supportList[-1])
195 |             if currentindex == upperindex-1 or currentindex==lowerindex:
196 |                 return bestcandidate,score
197 |         elif score < cutoff:
198 |             # In this case, cutoff is an upper bounf for the max score
199 |             upperindex=currentindex
200 |         else:
201 |             # If they are magically equal, this is the score
202 |             return bestcandidate,score
203 |         currentindex=(lowerindex + upperindex) // 2
204 | 
205 | def insertWithScore(a,candidate,cutoff):
206 |     oldcansupport=a.cansupport.copy()
207 |     a.elect(candidate)
208 |     for nom in a.voterlist:
209 |         for newedge in nom.edges:
210 |             if newedge.canindex== candidate.index:
211 |                 usedbudget = sum([a.edgeweight[edge.index] for edge in nom.edges])
212 |                 a.setweight(newedge, nom.budget-usedbudget)
213 |                 for edge in nom.edges:
214 |                     if edge.canindex != candidate.index and a.edgeweight[edge.index] > 0.0:
215 |                         if oldcansupport[edge.canindex] > cutoff:
216 |                             fractiontotake = cutoff / oldcansupport[edge.canindex]
217 |                             a.setweight(newedge, a.edgeweight[newedge.index] + a.edgeweight[edge.index]* fractiontotake)
218 |                             a.setweight(edge, a.edgeweight[edge.index] * (1-fractiontotake))
219 | 
220 | def approvalvoting(votelist,numtoelect):
221 |     nomlist,candidates=setuplists(votelist)
222 |     #creating an assignment now also computes the total possible stake for each candidate
223 |     a=assignment(nomlist,candidates)
224 | 
225 |     candidatessorted=sorted(candidates, key = lambda x : a.canapproval[x.index], reverse=True)
226 |     for candidate in candidatessorted[0:numtoelect]:
227 |         a.elect(candidate)
228 |     for nom in a.voterlist:
229 |         numbelected=len([edge for edge in nom.edges if a.canelected[edge.canindex]])
230 |         if (numbelected > 0):
231 |             for edge in nom.edges:
232 |                 a.setweight(edge,nom.budget/numbelected)
233 |     return a
234 | 
235 | def printresult(a,listvoters=True,listelectedcandidates=True):
236 |     if listelectedcandidates:
237 |         for candidate in a.electedcandidates:
238 |             print(candidate.canid," is elected with stake ",a.cansupport[candidate.index], "and score ",a.canscore[candidate.index])
239 |         print()
240 |     if listvoters:
241 |         for nom in a.voterlist:
242 |             print(nom.voterid," has load ",a.voterload[nom.index], "and supported ")
243 |             for edge in nom.edges:
244 |                 print(edge.canid," with stake ",a.edgeweight[edge.index], end=" ")
245 |             print()
246 |     print("Minimum support ",min([a.cansupport[candidate.index] for candidate in a.electedcandidates]))
247 | 
248 | def equalise(a, nom, tolerance):
249 |     # Attempts to redistribute the nominators budget between elected validators
250 |     # Assumes that all elected validators have backedstake set correctly
251 |     # returns the max difference in stakes between sup
252 | 
253 |     electededges=[edge for edge in nom.edges if a.canelected[edge.canindex]]
254 |     if len(electededges)==0:
255 |         return 0.0
256 |     stakeused = sum([a.edgeweight[edge.index] for edge in electededges])
257 |     backedstakes=[a.cansupport[edge.canindex] for edge in electededges]
258 |     backingbackedstakes=[a.cansupport[edge.canindex] for edge in electededges if a.edgeweight[edge.index] > 0.0]
259 |     if len(backingbackedstakes) > 0:
260 |         difference = max(backingbackedstakes)-min(backedstakes)
261 |         difference += nom.budget-stakeused
262 |         if difference < tolerance:
263 |             return difference
264 |     else:
265 |         difference = nom.budget
266 |     #remove all backing
267 |     for edge in nom.edges:
268 |         a.setweight(edge, 0.0)
269 |     electededges.sort(key=lambda x: a.cansupport[x.canindex])
270 |     cumulativebackedstake=0
271 |     lastcandidateindex=len(electededges)-1
272 |     for i in range(len(electededges)):
273 |         backedstake=a.cansupport[electededges[i].canindex]
274 |         #print(nom.nomid,electededges[i].valiid,backedstake,cumulativebackedstake,i)
275 |         if backedstake * i - cumulativebackedstake > nom.budget:
276 |             lastcandidateindex=i-1
277 |             break
278 |         cumulativebackedstake +=backedstake
279 |     laststake=a.cansupport[electededges[lastcandidateindex].canindex]
280 |     waystosplit=lastcandidateindex+1
281 |     excess = nom.budget + cumulativebackedstake -  laststake*waystosplit
282 |     for edge in electededges[0:waystosplit]:
283 |         a.setweight(edge,excess / waystosplit + laststake - a.cansupport[edge.canindex])
284 |     return difference
285 | 
286 | import random
287 | def equaliseall(a,maxiterations,tolerance,debug=False):
288 |     for i in range(maxiterations):
289 |         for j in range(len(a.voterlist)):
290 |             nom=random.choice(a.voterlist)
291 |             equalise(a,nom,tolerance/10)
292 |         maxdifference=0
293 |         for nom in a.voterlist:
294 |             difference=equalise(a,nom,tolerance/10)
295 |             maxdifference=max(difference,maxdifference)
296 |         if maxdifference < tolerance:
297 |             if debug:
298 |                 print("max iterations ",maxiterations," actual iterations ",i+1)
299 |             return
300 |     if maxiterations > 1 or debug:
301 |         print(" reached max iterations ",maxiterations)
302 | 
303 | def seqPhragménwithpostprocessing(votelist,numtoelect, ratio=1):
304 |     a = seqPhragmén(votelist,numtoelect)
305 |     passes=math.floor(ratio*numtoelect)
306 |     equaliseall(a,ratio*numtoelect,0.1, True)
307 |     return a
308 | 
309 | 
310 | def factor3point15(votelist, numtoelect,tolerance=0.1):
311 |     nomlist,candidates=setuplists(votelist)
312 |     a=assignment(nomlist,candidates)
313 | 
314 |     for round in range(numtoelect):
315 |         bestcandidate,score=calculateMaxScore(a)
316 |         insertWithScore(a,bestcandidate, score)
317 |         equaliseall(a,1000000,tolerance)
318 |     return a
319 | 
320 | def maybecandidate(a,newcandidate,shouldremoveworst, tolerance):
321 |     assert(a.canelected[newcandidate.index]==False)
322 |     currentvalue=min([a.cansupport[candidate.index] for candidate in a.electedcandidates])
323 |     #To find a new assignment without losing our current one, we will need to copy the edges
324 |     b=assignment(a.voterlist,a.candidates,a)
325 |     if shouldremoveworst:
326 |         worstcanidate =min(electedcandidates, key = lambda x: b.cansupport[x.index])
327 |         b.unelect(worstcandidate)
328 |     b.elect(newcandidate)
329 |     equaliseall(b,100000000,tolerance)
330 |     newvalue=min([b.cansupport[candidate.index] for candidate in b.electedcandidates])
331 |     return b, newvalue
332 | 
333 | def SFFB18(votelist, numtoelect,tolerance=0.1):
334 |     nomlist,candidates=setuplists(votelist)
335 |     a=assignment(nomlist,candidates)
336 |     for round in range(numtoelect):
337 |         if round == 0:
338 |             newcandidate=max([(can,a.canapproval[can.index]) for can in a.candidates],key = lambda x : x[1])[0]
339 |             a.elect(newcandidate)
340 |             equaliseall(a,1,tolerance)
341 |         else:
342 |             bestvalue=0
343 |             for can in a.candidates:
344 |                 if not a.canelected[can.index] and a.canapproval[can.index] > bestvalue:
345 |                     b,newvalue = maybecandidate(a,can, False, tolerance)
346 |                     if newvalue > bestvalue:
347 |                         bestassignment=b
348 |                         bestvalue=newvalue
349 |             if bestvalue > 0:
350 |                 a=bestassignment
351 |     return a
352 | 
353 | def binarysearchfeasible(votelist,numtoelect,tolerance=0.1):
354 |     nomlist,candidates=setuplists(votelist)
355 |     a=assignment(nomlist,candidates)
356 | 
357 |     #First do factor 3.15
358 |     #but keep track of the order we elect people and the value then
359 |     orderelectedwithvalue=[]
360 |     for round in range(numtoelect):
361 |         bestcandidate,score=calculateMaxScore(a)
362 |         insertWithScore(a,bestcandidate, score)
363 |         equaliseall(a,1000000,tolerance/numtoelect)
364 |         currentvalue=min([a.cansupport[candidate.index] for candidate in a.electedcandidates])
365 |         orderelectedwithvalue.append((bestcandidate,currentvalue))
366 |     if len(a.candidates)==numtoelect:
367 |         return a
368 |     bestknownvalue=currentvalue
369 |     bestassignment=assignment(a.voterlist,a.candidates,a)
370 |     bestorderelected=orderelectedwithvalue.copy()
371 |     maxunelectedapproval = max([a.canapproval[i] for i in range(len(a.candidates)) if not a.canelected[i]])
372 |     totalvotes=sum([nom.budget for nom in a.voterlist])
373 |     maxvalue=min(3.15*currentvalue, max(currentvalue,maxunelectedapproval), totalvotes/numtoelect)
374 |     while(maxvalue - bestknownvalue > tolerance):
375 |         targetvalue=math.sqrt(maxvalue*bestknownvalue)
376 |         lastgoodindex=len([x for x in orderelectedwithvalue if x[1] >= targetvalue])-1
377 |         #print(orderelectedwithvalue, targetvalue,lastgoodindex, maxvalue,bestknownvalue,currentvalue,targetvalue)
378 |         assert(lastgoodindex >= 0)
379 |         assert(orderelectedwithvalue[lastgoodindex][1] >= targetvalue)
380 |         for x in orderelectedwithvalue[lastgoodindex+1:]:
381 |             a.unelect(x[0])
382 |         del orderelectedwithvalue[lastgoodindex+1:]
383 |         for nom in a.voterlist:
384 |             for edge in nom.edges:
385 |                 if not a.canelected[edge.canindex]:
386 |                     a.setweight(edge,0)
387 |         equaliseall(a,1000000,tolerance/numtoelect)
388 |         currentvalue=min([a.cansupport[candidate.index] for candidate in a.electedcandidates])
389 |         if currentvalue < targetvalue:
390 |             if targetvalue >= sqrt(currentvalue, maxvalue):
391 |                 #At this point we are getting so much error from tolerance in equaliseall
392 |                 #that we should give up
393 |                 print("Giving up with error at most ",maxvalue-bestknownvalue)
394 |                 return
395 |             else:
396 |                 targetvalue=currentvalue
397 |         #print(targetvalue,lastgoodindex, maxvalue,bestknownvalue,currentvalue)
398 | 
399 | 
400 |         for round in range(lastgoodindex+1,numtoelect):
401 |             # First try maxscore candidate, which will help with PJR
402 |             bestcandidate,score=calculateMaxScore(a)
403 |             if score >= targetvalue:
404 |                 insertWithScore(a,bestcandidate, score)
405 |                 equaliseall(a,1000000,tolerance/numtoelect)
406 |                 currentvalue=min([a.cansupport[candidate.index] for candidate in a.electedcandidates])
407 |                 assert(currentvalue >= targetvalue)
408 |                 orderelectedwithvalue.append((bestcandidate,currentvalue))
409 |                 continue
410 |             else:
411 |                 b,newvalue = maybecandidate(a,bestcandidate, False, tolerance)
412 |                 if newvalue >= targetvalue:
413 |                     a=b
414 |                     orderelectedwithvalue.append((bestcandidate,newvalue))
415 |                     currentvalue=newvalue
416 |                     continue
417 |             #Then try some candidates in which we are guaranteed that one is feasible if threshold >= d*/2
418 |             calculateScores(a,targetvalue/2)
419 |             scores=[(can, a.canscore[can.index]) for can in a.candidates if not a.canelected[can.index] and a.canapproval[can.index] >= targetvalue and a.canscore[can.index] >= targetvalue/2]
420 |             scores.sort(reverse=True,key=lambda x: x[1])
421 |             for can,score in scores:
422 |                 b,newvalue = maybecandidate(a,can, False, tolerance)
423 |                 if newvalue >= targetvalue:
424 |                     a=b
425 |                     orderelectedwithvalue.append((can,newvalue))
426 |                     currentvalue=newvalue
427 |                     break
428 |             else:
429 |                 break
430 |         # print("here",currentvalue,targetvalue)
431 |         if len(a.electedcandidates) < numtoelect:
432 |             maxvalue = targetvalue
433 |             a=bestassignment
434 |             orderelectedwithvalue=bestorderelected
435 |         else:
436 |             bestknownvalue=currentvalue
437 |             bestassignment=assignment(a.voterlist,a.candidates,a)
438 |             bestorderelected=orderelectedwithvalue.copy()
439 |     return a
440 | 
441 | import unittest
442 | class electiontests(unittest.TestCase):
443 |     def testexample1Phragmén(self):
444 |         votelist=[("A",10.0,["X","Y"]),("B",20.0,["X","Z"]),("C",30.0,["Y","Z"])]
445 |         a = seqPhragmén(votelist,2)
446 |         self.assertEqual({can.canid for can in a.electedcandidates},{"Y","Z"})
447 |         self.assertAlmostEqual(a.canscore[2],0.02)
448 |         self.assertAlmostEqual(a.canscore[1],0.04)
449 |     def testexample1approval(self):
450 |         votelist=[("A",10.0,["X","Y"]),("B",20.0,["X","Z"]),("C",30.0,["Y","Z"])]
451 |         a = approvalvoting(votelist,2)
452 |         self.assertEqual({can.canid for can in a.electedcandidates},{"Y","Z"})
453 |         self.assertAlmostEqual(a.canapproval[2],50.0)
454 |         self.assertAlmostEqual(a.canapproval[1],40.0)
455 | def dotests():
456 |     unittest.main()
457 | 
458 | import time
459 | def doall(votelist, numtoelect, listvoters=True, listcans=True):
460 |     if listvoters:
461 |         print("Votes ",votelist)
462 |     alglist=[(approvalvoting,"Approval voting"), (seqPhragmén, "Sequential Phragmén"),
463 |              (seqPhragménwithpostprocessing, "Sequential Phragmén with post processing"),
464 |              (factor3point15, "The factor 3.15 thing"), (binarysearchfeasible,"Factor 2 by binary search"), (SFFB18, "SFFB18")]
465 |     for alg,name in alglist:
466 |         st=time.perf_counter()
467 |         a = alg(votelist,numtoelect)
468 |         et=time.perf_counter()
469 |         print(name, " gives")
470 |         printresult(a,listvoters,listcans)
471 |         print(" in ",et-st," seconds.")
472 |         print()
473 | 
474 | 
475 | def example1():
476 |     votelist=[("A",10.0,["X","Y"]),("B",20.0,["X","Z"]),("C",30.0,["Y","Z"])]
477 |     doall(votelist,2)
478 | 
479 | 
480 | def example2():
481 |     # Approval voting does not do so well for this kind of thing.
482 |     votelist=[("A",30.0,["T", "U","V","W"]),("B",20.0,["X"]),("C",20.0,["Y"]),("D",20.0,["Z"])]
483 |     doall(votelist,4)
484 | 
485 | def example3():
486 |     #Proportional representation test.
487 |     #Red should has 50% more votes than blue. So under PR, it would get 12/20 seats
488 |     redparty=["Red"+str(i) for i in range(20)]
489 |     blueparty=["Blue"+str(i) for i in range(20)]
490 |     redvoters = [("RedV"+str(i),20.0,redparty) for i in range(30)]
491 |     bluevoters = [("BlueV"+str(i),20.0,blueparty) for i in range(20)]
492 |     votelist= redvoters+bluevoters
493 |     doall(votelist, 20, False)
494 | 
495 | 
496 | def example4():
497 |     #Now we want an example where seq Phragmén is not so good.
498 |     votelist=[("A",30.0,["V","W"]),("B",20.0,["V","Y"]),("C",20.0,["W","Z"]),("D",20.0,["Z"])]
499 |     print("Votes ",votelist)
500 |     doall(votelist,4)
501 | 
502 | def example5():
503 |     votelist = [
504 | 		("10", 1000, ["10"]),
505 | 		("20", 1000, ["20"]),
506 | 		("30", 1000, ["30"]),
507 | 		("40", 1000, ["40"]),
508 | 		('2', 500, ['10', '20', '30']),
509 | 		('4', 500, ['10', '20', '40'])
510 | 	]
511 |     print("Votes ",votelist)
512 |     doall(votelist,4)
513 | 
514 | def example6():
515 |     #Now we want an example where seq Phragmén is not so good.
516 |     votelist=[("A",100.0,["V","W","X","Y","Z"]),("B",100.0,["W","X","Y","Z"]),
517 |               ("C",100.0,["X","Y","Z"]),("D",100.0,["Y","Z"]), ("E",100.0,["Z"]),
518 |               ("M",50.0, ["M"])]
519 |     print("Votes ",votelist)
520 |     doall(votelist,5)
521 | 
522 | def exampleLine():
523 |     votelist = [
524 | 		("a", 2000, ["A"]),
525 | 		("b", 1000, ["A","B"]),
526 | 		("c", 1000, ["B","C"]),
527 | 		("d", 1000, ["C","D"]),
528 | 		("e", 1000, ["D","E"]),
529 | 		("f", 1000, ["E","F"]),
530 |                 ("g", 1000, ["F","G"])
531 | 	]
532 |     doall(votelist,7)
533 | 
534 | def ri(vals=20,noms=2000, votesize=10):
535 |     #Let's try a random instance
536 |     candidates=["Val"+str(i) for i in range(vals)]
537 |     votelist=[("Nom"+str(i), 100, random.sample(candidates,votesize)) for i in range(noms)]
538 |     doall(votelist, vals // 2, False, False)
539 | 
540 | def ripartylist(vals=200,noms=2000, votesize=10,seed=1):
541 |     #Half the validators are in a party which 1/4 of the nominators vote for.
542 |     # Approval voting does worse now
543 |     # and this is probably more realistic than the pure random instance.
544 |     random.seed(seed)
545 |     candidates=["Val"+str(i) for i in range(vals//2)]
546 |     partycandidates=["PartyVal"+str(i) for i in range(vals- vals // 2)]
547 |     partynoms = noms // 4
548 |     votelist=[("Nom"+str(i), 100, random.sample(candidates,votesize)) for i in range(noms - partynoms)]
549 |     votelist += [("Nom"+str(i), 100, partycandidates) for i in range(partynoms)]
550 |     return votelist
551 | 
552 | 
553 | def riparty(vals=200,noms=2000, votesize=10,seed=1):
554 |     #Half the validators are in a party which 1/4 of the nominators vote for.
555 |     # Approval voting does worse now
556 |     # and this is probably more realistic than the pure random instance.
557 |     votelist=ripartylist(vals,noms,votesize,seed)
558 |     doall(votelist, vals // 4, False, False)
559 | 
560 | 
561 | 
562 | 
563 | 
564 | 
565 | 
566 | 
567 | 
568 | 
569 | 
570 | 
571 | 
572 | 
573 | 
574 | 
575 | 
576 | 
577 | 
578 | 
579 | 
580 | 
581 | 
582 | 
583 | 
584 | 
585 | 
586 | 
587 | 
588 | 


--------------------------------------------------------------------------------
/Polkadot_Finality_Gadget_v9000.md:
--------------------------------------------------------------------------------
  1 | # Polkadot Finality Gadget (AFG) v9000
  2 | 
  3 | We have a chain selection rule and block production mechanism that probably gives us consensus on a prefix of the chain including all but the most recent blocks. However we need to prove to 3rd parties that far enough back blocks are final by producing a certificate signed by $2/3$ of validators.
  4 | 
  5 | The idea is to run a BFT agreement algorithm similar to Tendermint or Algorand agreement but on chains rather than individual blocks so we do not need to reach agreement on each block. The algorithm will have a primary but their role will be to coordinate locking rather than propose blocks. Instead everyone will vote on their best block.
  6 | 
  7 | We can consider a prevote or precommit for a block as a vote for every block on that chain not already finalised. Thus we may have $2/3$ votes on many blocks in one vote but it is easy to see that all blocks with $2/3$ votes are in one chain.
  8 | 
  9 | We will want to change the set of participants who actively agree sometimes. To model this, we have a large set of participants who follow messages. For each voting step, there is a set of $n$ voters. We will frequently need to assume that for each such step, at most  $f < n/3$ voters are faulty. We need $n-f$ of voters to agree on finality. Whether or not block producers ever vote, they will need to be participants who track the state of the protocol.
 10 | 
 11 | Participants remember which block they see as currently being the latest finalised block and a chain they are locked to. This locked chain represents an estimate of which block could have been finalised already.
 12 | 
 13 | **Rounds**: each participant has their own idea of what the current round number is. Every prevote and precommit has an associated round number. Honest voters only vote once (for each type of vote) in each round and don't vote in earlier rounds after later ones.
 14 | 
 15 | Each round has two phases, each of which has an associated vote, prevote and precommit.
 16 | 
 17 | ### Preliminaries
 18 | 
 19 | For block $B$, we write $\mathrm{chain}(B)$ for the chain whose head is $B$. The block number, $n(B)$ of a block $B$ is the length of $\mathrm{chain}(B)$.
 20 | 
 21 | For blocks $B'$, $B$, $B$ is later than $B'$ if it has a higher block number.
 22 | We write $B > B'$ or that $B$ is descendent of $B'$ for $B$, $B'$ appearing in the same blockchain with $B'$ later i.e. $B \in \mathrm{chain}(B')$ with $n(B') > n(B)$ and $B < B'$ or $B$ is an ancestor of $B'$ for $B' \in \mathrm{chain}(B)$ with $n(B) > n(B')$ . $B \geq B'$ an $B \leq B'$ are similar except allowing $B = B$. We write $B \sim B'$ or $B$ and $B'$ are on the same chain if $B<B'$,$B=B'$ or $B> B'$ and $B \nsim B'$ or $B$ and $B'$ are not on the same chain if there is no such chain.
 23 | 
 24 | Blocks are ordered as a tree with the genesis block as root. So any two blocks have a common ancestor but two blocks not on the same chain do not have a common descendent.
 25 | 
 26 | A vote $v$ for a block $B$ by a validator $V$ is a message signed by $V$ containing the blockhash of $B$ and meta information like the round numbers and the type of vote. 
 27 | 
 28 | We call a set $S$ of votes tolerant if the number of validators with more than one vote in $S$ is at most $f$. We say that $S$ has supermajority for a block $B$ if the set of validators with votes for blocks $\geq B$ has size at least $(n+f+1)/2$.
 29 | 
 30 | The $2/3$-GHOST function $g(S)$ takes a set $S$ of votes and returns the block $B$ with highest block number such that $S$ has a supermajority for $B$. If there is no such block, then it returns `nil`. (if $f \neq \lfloor (n-1)/3 \rfloor$, then this is a misnomer and we may change the name accordingly.)
 31 | 
 32 | Note that, if $S$ is tolerant, then we can compute $g(S)$ by starting at the genesis block and iteratively looking for a child of our current block with a supermajority, which must be unique if it exists. Thus we have:
 33 | **Lemma 1**: Let $T$ be a tolerant set of votes. Then
 34 | 
 35 | 1. The above definition uniquely defines $g(T)$
 36 | 2. If $S \subseteq T$ has $g(S) \neq$ nil, then $g(S) \leq g(T)$.
 37 | 3. If $S_i \subseteq T$ for $1 \leq i \leq n$ then all non-nil $g(S_i)$ are on a single chain with head $g(T)$.
 38 | 
 39 | Note that we can easily update $g(S)$ to $g(S \cup \{v\})$, by checking if any child of $g(S)$ now has a supermajority.
 40 | 
 41 | 3 tells us that even if validators see different subsets of the votes cast in a given voting round, this rule may give them different blocks but all such blocks are in the same chain under this assumption. 
 42 | 
 43 | We say that it is possible for a set $S$ to have a supermajority for $B$ if $2f+1$ validators either vote for a block $\not \geq B$ or equivocate in $S$. Note that if $S$ is tolerant, it is possible for $S$ to have a supermajority for $B$ if and only if there is a tolerant $T \supseteq S$ that has a supermajority for $B$.
 44 | 
 45 | 
 46 | We say that it is impossible for any child of $B$ to have a supermajority in $S$ if $S$ has votes from at least $2f+1$ validators and it is impossible for $S$ to have a supermajority for each child of $B$ appearing on the chain of any vote in $S$. Again, provided $S$ is tolerant, this holds if and only if for any possible child of $B$, there is no tolerant $T \subset S$ that has a supermajority for that child.
 47 | 
 48 | Note that it is possible for an intolerant $S$ to both have a supermajority for $S$ and for it to be impossible to have such a supermajority under these definitions, as we regard such sets as impossible anyway.
 49 | 
 50 | **Lemma 2**
 51 | (i) If $B' \geq B$ and it is imposible for $S$ to have a supermajority for $B$, then it is imposible for $S$ to have a supermajority for $B'$.
 52 | (ii) If $S \subseteq T$ and it is impossible for $S$ to have a supermajority for $B$
 53 | (iii) If $g(S)$ exists and $B \nsim g(S)$ then it is impossible for $S$ to have a supermajority for $B$.
 54 | 
 55 | ## Algorithm
 56 | 
 57 | We let $V_{r,v}$ and $C_{r,v}$ be the sets of prevotes and precommits respectively recieved by $v$ from round $r$ at the current time.
 58 | 
 59 | We define $E_{r,v}$, $v$'s estimate of what might have been finalised in round $r$, to be the last block in the chain with head $g(V_{r,v})$ that it is possible for $C_{r,r}$ to have a supermajority for. If either $E_{r,v} < g(V_{r,v})$ or it is impossible for $C_{r,v}$ to have a supermajority for any children of $g(V_{r,v})$,, then we say that ($v$ sees that) round $r$ is completable. $E_{0,v}$ is the genesis block (if we start at $r=1$).
 60 | 
 61 | We have a time bound $T$, that we hope is enough to send messages and gossip them to everyone. 
 62 | 
 63 | In round $r$ an honest validator $v$ does the following:
 64 | 
 65 | 1. $v$ can start round $r > 1$ when round $r-1$ is completable and $v$ has cast votes in all previous rounds where they are a voter. Let $t_{r,v}$ be the time we start round $r$.
 66 | 
 67 | 2. At time $t_{r,v}$, if $v$ ais the primary of this round and has not finalised $E_{r-1,v}$ then they broadcast $E_{r-1,v}$. If thy have finalised it, they can broadcast $E_{r-1,v}$ anyway (but don't need to).
 68 | 
 69 | 3. If $v$ is a voter for the prevote of round $r$, $v$ waits until either it is at least time $t_{r,v}+2T$ or round $r$ is completable, then broadcasts a prevote. They prevote for the head of the best chain containing $E_{r-1,v}$ unless we recieved a block $B$ from the primdary and $g(V_{r-1,v}) \geq B > E_{r-1,v}$, in which case they use the best chain containing $B$ instead.
 70 | 
 71 | 4. If $v$ is a voter for the precommit step in round $r$, then they wait until $g(V_{r,v}) \geq E_{r-1,v}$ and one of the following  conditions holds
 72 | (i) it is at least time $t_{r,v}+4T$, 
 73 | (ii) round $r$ is completable or
 74 | (iii) it is impossible for $V_{r,v}$ to have a supermajority for any child of $g(V_{r,v})$,
 75 | and then broadcasts a precommit for $g(V_{r,v})$ _( (iii) is optional, we can get away with just (i) and (ii))_.
 76 | 
 77 | 
 78 | ### Finalisation
 79 | 
 80 | If, for some round $r$, at any point after the precommit step of round $r$, we have that $B=g(C_{r,v})$ is later than our last finalised block and $V_{r,v}$ has a supermajority, then we finalise $B$. We may also send a commit message for $B$ that consists of $B$ and a set of precommits for blocks $\geq B$ (ideally for $B$ itself if possible see "Alternatives to the last blockhash" below). 
 81 | 
 82 | To avoid spam, we only send commit messages for $B$ if we have not receive any valid commit messages for $B$ and its decendents and we wait some time chosen uniformly at random from $[0,1]$ seconds or so before broacasting.
 83 | 
 84 | If we recieve a valid commit message for $B$ for round $r$, then it contains enough precommits to finalise $B$ itself if we haven't already done so, so we'll finalise $B$ as long as we are past the precommit step of round $r$.
 85 | 
 86 | # Analysis
 87 | 
 88 | ## Accountable Safety
 89 | 
 90 | The first thing we want to show is asynchronous safety if we have at most $f$ Byzantine validators:
 91 | 
 92 | **Theorem 1**: If the protocol finalises any two blocks $B,B'$ that have valid commit messages sent are on the same chain, then there are at least $f+1$ Byzantine voters who all voted in a particular vote. Furthermore, there is a synchronous procedure to find such a set.
 93 | 
 94 | The challenge procedure works as follows: If $B$ and $B'$ are committed in the same round, then the union of their precommits must contain at least $f$ equivocations, so we are done. Otherwise $B$ was commited in round $r$ and $B'$ in round $r' > r$ say. Then we ask the at least $n-f$ validators who precomitted $\geq B'$ in round $r$ in the commit message, why they precommited.
 95 | 
 96 | We ask queries of the following form: 
 97 | 
 98 | - Why was $E_{r''-1} \not\geq B$  when you prevoted for or precomitted to $B'' \not\geq B$ in round $r'' > r$?
 99 | 
100 | Which any honest validator should be able to respond to as is shown in Lemma **3** below. 
101 | 
102 | The response is of the following form:
103 | 
104 | - A either a set $S$ of prevotes for round $r''-1$ or a set $S$ of precommits for round $r''-1$  or  such that it is impossible for $S$ to have a supermajority for $B$.
105 | 
106 | If no validator responds, then we  have $n-f$ Byzantine validators. If any do, then if $r'' > r+1$,  we can ask the same query for $n-f$ validators in round $r''-1$.
107 | 
108 | If any responded and $r''=r+1$, then we have either a set $S$ of prevotes or precommits in round $r$ that it is impossible for $S$ to have a supermajority for $B$ in round $r$.
109 | 
110 | If $S$ is a set of precommits, then if we take the union of $S$ and the set of precommits in the commit message for $B$, then the resulting set of precommits for round $r$ has a supermajority for $B$ and it is impossible for it to have a supermajority for $B$. This is possible if the set is not tolerant and so there must be at least $f+1$ voters who equivocate an so are Byzantine.
111 | 
112 | If we get a set $S$ of prevotes for round $r$ that does not have a supermajority for $B$, then we need to ask a query of the form
113 | 
114 | - Which prevotes for round $r$ have you seen?
115 | 
116 | to all the voters of precommit in the commit message for $B$  who voted for blocks $B'' \geq B$. There must be $n-f$ such validators and a valid respone to this query is a set $T$ of prevotes for round $r$ with a supermajority for $B''$ and so a supermajority for $B$.
117 | 
118 | If any give a valid response, by a similar argument to the above, $S \union T$ will have $f+1$ equivocations.
119 | 
120 | So we either discover $f+1$ equivocations in a vote or else $n-f > f+1$ voters fail to validly respond like a honest voter could do to a query.
121 | 
122 | **Lemma 3** An honest validator can answer the first type of query.
123 | 
124 | We first need to show that for any prevote or precommit in round $r$ cast by an honest validator $v$ for a block $B''$, at the time of the vote we had $B'' \geq E_{r-1,v}$. Prevotes should be for the head of a chain containing either $E_{r-1,v}$ or $g(V_{r,v})$ Since $g(V_{r,v}) \geq E_{r-1},v$, in either case we have $B'' \geq E_{r-1,v}$. Precommits should be for $g(V_{r,v})$ but $v$ waits until $g(V_{r,v}) \geq E_{r-1,v}$ befor precommiting so again this holds.
125 | 
126 | Thus if $B'' \not\geq B$, then we had $E_{r-1,v} \not\geq B$.
127 | Next we need to show that if we had $E_{r-1,v} \not\geq B$ at the time of the vote then we can respond to the query validly. If $B$ wasn't on the same chain with $g(V_{r-1,v})$, then by Lemma 2 (iii), it was impossible for $V_{r-1,v}$ to have a supermajority for $B$. If it was on the same chain as $g(V_{r,-1v})$, then it was on the same chain as $E_{r-1,v}$ as well. Since $E_{r-1,v} \not\geq B$, in this case we must have $B > E_{r-1,v}$. However, possibly using that round $r-1$ is completable, it was impossible for $C_{r-1,v}$ to have a supermajority for any child of $E_{r-1,v}$ on the same chain with $g(V_{v,r})$ and in particular for the child of $E_{r-1,v}$ on $\textrm{chain}(B)$. By Lemma 2 (i), this means $C_{r-1,v}$ did not have a supermajority for $B$.
128 | 
129 | Thus we have that, at the time of the vote, for one of $V_{r-1,v}$, $C_{r-1,v}$, it was imposible to have a supermajority for $B$. The current sets $V_{r-1,v}$ and $C_{r-1,v}$ are supersets of those at the time of the vote, and so by Lemma 2 (ii), it is still impossible. Thus $v$ can respond validly.
130 | 
131 | 
132 | This is enough to show Theorem 1. Not that if $v$ sees a commit message for a block $B$ in round $r$ and has that $E_{r',v} \not\geq B$, for some completable round $r' \geq r$, then they should also be able to start a challenge procedure that succesfully identifies at least $f+1$ Byzantine voters in some round. Thus we have that:
133 | 
134 | **Lemma 4**
135 | If there at most $f$ Byzantine voters in any vote, $B$ was finalised in round $r$ and an honest participant $v$ sees that round $r' \geq r$ is completable, then $E_{r',v} \geq B$.
136 | 
137 | ## Liveness
138 | 
139 | We show the protocol is deadlock free and also that it finalises new blocks quickly in a weakly synchronous model.
140 | 
141 | Let's define $V_{r,v,t}$ be the set $V_{r,v}$ at time $t$ and similarly for $C_{r,v,t}$ and the block $E_{r,v,t}$ .
142 | 
143 | 
144 | **Lemma 5** Let $v,v'$ be (possibly identical) honest particpants, $t,t'$ be times and $r$ be a round. Then if $V_{r,v,t} \subseteq V_{r,v',r'}$ and $C_{r,v,t} \subseteq C_{r,v',r'}$, all these sets are tolerant and $v$ sees that $r$ is completable at time $t$, then $E_{r,v,t} \leq E_{r,v',t'}$ and $v'$ sees that $r$ is completable at time $t'$.
145 | 
146 | **Proof:** Since $v$ sees that $r$ is completable at time $t$, $V_{r,v,t}$, $C_{r,v,t}$ each contain votes from $n-f$ voters and so the same holds for $V_{r,v',t'}$ and $C_{r,v',t'}$. 
147 | By Lemma 1, $g(V_{r,v',t'}) \geq g(V_{r,v,t})$. Using Lemma 2, since it is impossible for $C_{r,v,t}$ to have a supermajority for any children of $g(V_{r,v,t}$, it is impossible for $C_{r,v',t'})$ as well and so $E_{r,v',t'} \leq g(V_{r,v,t})$. But now $E_{r,v,t}$,$E_{r,v',t'}$ are the last blocks on $\textrm{chain}(g(V_{r,v,t}))$ that it is possible for $C_{r,v,t},C_{r,v',t'}$ respectively to have a supermajority for. Thus by Lemma 2 (ii), $E_{r,v',t'} \leq E_{r,v,t}$.
148 | 
149 | ### Deadlock Freeness
150 | 
151 | Now we can show deadlock freeness for the asynchronous gossip network model, when a message that is sent or recieved by any honest participant is eventually recieved by all honest participants.
152 | 
153 | **Proposition 6** Suppose that we are in the aynchonous gossip network model and that at most $f$ voters for any vote are Byzantine. Then the protocol is deadlock free.
154 | 
155 | **Proof:** We need to show that if all honest participants reach some vote, then all of them eventually reach the next.
156 | 
157 | If all honest voters reach a vote, then they will vote and all honest participants see their votes. We need to deal with the two conditions that might block the algorithm even then. To reach the prevote of round $r$, a particpant may be held up at the condition that round $r-1$ must be completable. To reach the precommit, a voter may be held up by the condition that $g(V_{r,v}) \geq E_{r-1,v}$.
158 | 
159 | For the first case, the prevote, let $S$ be the set of all prevotes from round $r-1$ that any honest voter saw before they precommitted in round $r-1$. By Lemma 1, when voter $v'$ precommitted, they do it for block $g(V_{r-1,v'}) \leq g(S)$. Let $T$ be the set of precommits in round $r$ cast by honest voters. Then or any block $B \not\leq g(S)$, $T$ does not contain any votes that are $\geq B$ and so it is impossible for $T$ to have a supermajority for $B$. In particular, it is impossible for $T$ to have a supermajority for any child of $g(S)$. 
160 | 
161 | Now consider a voter $v$. By our network assumption, there is a time $t$ by which they have seen the votes in $S$ and $T$. Consider any $t' \geq t$. At this point we have $g(V_{r,v,t;}) \geq g(S)$. It is impossible for $C_{r,v,t'}$ to have a supermajority for any child of $g(S)$ and so $E_{r-1,v,t'} \leq g(S)$, whether or not this inequality is strict, we satisfy one of the two conditions for $v$ to see that round $r-1$ is completable at time $t'$. Thus if all honest voters reach the precommit vote of round $r-1$, all honest voters reach the prevote of round $r$.
162 | 
163 | Now we consider the second case, reaching the precommit. Note that any honest prevoter in round $r$ votes for a block $B_v \geq E_{r-1,v,t_v}$ where $t_v$ is the time they vote. Now consider any honest voter for the precommit $v'$. By some time $t'$, they have recieved all the messages recieved by each honest voter $v$ at time $t_v$ and $v'$'s prevote. Then by Lemma 4, $B_v \geq E_{r-1,v,t_v} \geq E_{r-1,v',t'}$. Since $V_{r,v',t'}$ contains these $B_v$, $g(V_{r,v',t'}) \geq  E_{r-1,v',t'}$. Thus if all honest voters prevote in round $r$, eventually all honest voters precommit in round $r$.
164 | 
165 | An easy induction completes the proof of the proposition.
166 | 
167 | ### Weakly synchronous liveness
168 | 
169 | Now we consider the weakly synchronous gossip network model. The idea that there is some global stabilisation time($\textrm{GST}$) such that any message recieved or sent by an honest participant at time $t$ is recieved by all honest paticipants at time $\max\{t,\textrm{GST}\}+T$.
170 | 
171 | Let $t_r$ be the first time any honest particpant enters round $r$ i.e. the minimum over honest participants $v$ of $t_{r,v}$.
172 | 
173 | **Lemma 7** Assume the weakly synchronous gossip network model and that each vote has at most $f$ Byzantine voters. Then if $t_r \geq \textrm{GST}$, we have that 
174 | (i) $t_r \leq t_{r,v} \leq t_r+T$ for any honest particpant $v$,
175 | (ii) no honest voter prevotes before time $t_r+2T$,
176 | (iii) any honest voter $v$ precommits at the latest at time $t_{r,v}+4T$,
177 | (iv) for any honest $v$, $t_{r+1,v} \leq t_r + 6T$.
178 | 
179 | 
180 | **Proof:** Let $v'$ be one of the first valiators to enter round $r$ i.e. with $t_{r,v'}=t_r$. By our network assumption, all messages recieved by $v'$ beore they ended are recieved by all honest participants before time $t_r+T$. In particular at time $t_r$, $v'$ sees that all previous rounds are completable and so by Lemma 4, so does every other honest validator by time $t_r+T$. Also since for $r' < r$, at some time $s_{r'} \leq t_r$ $g(V_{r',v',s_r'}) \geq E_{r',v',s_r'}$, again by Lemma 4, for all honest $v$, $g(V_{r',v,t_r+T}) \geq E_{r',v,t_r+T}$. Looking at the conditions for voting, this means that any honest validator does not need to wait before voting in any round $r' \leq r$. Thus they cast any remaining votes and enter round $r$ by time $t_r + T$. This shows (i).
181 | 
182 | For (ii), note that the only reason why an honest voter would not wait until time $t_{r,v}+2T \geq t_r+ 2T$ is when $n-f$ voters have already prevoted. But since some of those $n-f$ votes are honest, this is impossible before $t_r+2T$
183 | 
184 | Now an honest voter $v''$ prevotes at time $t_{r,v''}+2T \leq t_r +3T$ and by our network assumptions all honest validators recieve this vote by time $t_r+4T$. An honest voter for the precommit $v$ has also recieved all messages that $v''$ recieved before they prevoted by then. Thus the block they prevoted has $B_{v''} \geq E_{r-1,v''} \geq E_{r-1,v,t_r+4T}$, since this holds for every honest voter $v''$, $g(V_{r,v,t_r+4T}) \geq E_{r-1,v,t_r+4T}$. Thus they will precommit by time $t_{r,v}+4T$ which shows (iii).
185 | 
186 | By the network assumption an honest voter $v'$'s precommit will be recieved by all honest validators $v$ by time $t_{r,v'}+ 5T \leq t_r+6T$. Since $v$ will also have recieved all prevotes $v$ say when they precommitted by this time, their vote $B_{v'}$ will have $B_{v'}=g(V_{r,v'}) \leq g(V_{r,v,t_r+6T})$. Thus $C_{r, v, t_r+6T}$ contains precommits from $n-f$ voters $v'$ with $B_{v'} \leq g(V_{r,v,t_r+6T})$ and thus it is impossible for $C_{r,v,t_r+6T}$ to have a supermajority for any children of $g(V_{r,v, t_r+6T})$. Thus $v$ sees that round $r$ is comletable at time $t_r+6T$. Since they have aleady prevoted and precommited if they were a voter, they will move to round $r+1$ by at latest $t_t+6T$. This is (iv).
187 | 
188 | 
189 | **Lemma 8**
190 | Suppose $t_r \geq GST$ and very vote has at most $f$ Byzanyine voters. Let $H_r$ be the set of prevotes ever cast by honest voters in round $r$. Then
191 | 
192 | (a) any honest voter precommits to a block $\geq  g(H_r)$,
193 | 
194 | (b) every honest paticipant finalises $g(H_r)$ by time $t_r+6T$.
195 | 
196 | **Proof:** For (a), we separate into cases based on which of the conditions (i)-(iii) that we wait for to precommit hold.
197 | 
198 | For (i), all honest voters prevote in round $r$ by time $t_r+3T$. So any honest voter $v$ who precommits at or after time $t_{r,v}+4T \geq t_r+4T$ has recieved all votes in $H_r$ and by Lemma 1, precommits to a block $\geq g(H_r)$.
199 | 
200 | For (ii), we argue that no honest voter commits a block $\not\geq g(H_r)$ first. The result will then follow by an easy induction once the other cases are dealt with. Suppose that no honest voter has precommited a block $\not \geq g(H_r)$ so far and that a voter $v$ votes early because of (ii).
201 | 
202 | Note that, since we assume that all precommits by honest voters so far were $\geq g(H_r)$, it is possible for $C_{r,v}$ to have a supermajority for $g(H_r)$. For (ii) to hold for a voter $v$ i.e for round $r$ to be completable, it must be the case that either it is impossible for $C_{r,v}$ to have a supermajority for $g(V_{r,v})$ or else be impossible for $C_{r,v}$ to have a supermajority for any children of $g(V_{r,v})$. By Lemma 2, we cannot have $g(V_{r,v}) < g(H_r)$. But by Lemma 1, these are on the same chain and so $g(V_{r,v}) \geq g(H_r)$. Since this is the block $v$ precommits to, we are oone in case (ii)
203 | 
204 | For (iii), let $v$ be the voter in question. Note that since $n-f$ honest voters prevoted $\geq g(H_r)$, it is possible for $V_{r,v}$ to have a supermajority for $g(H_r)$. By Lemma 1, $g(V_{r,v})$ is on the same chain as $g(H_r)$. For (iii), it is impossible for $V_{r,v}$ to have a supermajority for any children of $g(V_{r,v})$. If we had $g(V_{r,v}) < g(H_r)$, by Lemma 2, this would mean that it would be impossible for $V_{r,v}$ to have a supermajority for $g(H_r)$ as well. So it must be that $g(V_{r,v} )\geq g(H_r)$ as required.
205 | 
206 | For (b), combining (a) and Lemma 7 (iii), we have that any honest voter $v$ precommits $\geq g(H_r)$ by time $t_{r,v}+4T$. By our netork assumption, all honest particpants recieve these precommits by time $t_r+6T$ and so finalis $g(H_r)$ if they have not done so already.
207 | 
208 | 
209 | **Lemma 9** Suppose that $t_r \geq GST$, the primary $v$ of round $r$ is honest and no vote has more than $f$ Byzantine voters. Let $B=E_{r-1,v,t_{v,r}}$ be the block $v$ broacasts if it is not final. Then every honest prevoter prevotes for the best chain including $B$ and all honest voter finalise $B$ by time $t_r+6T$.
210 | 
211 | **proof**: By Lemma 7 and our nwtork asumptions, no honet voter  prevotes befor time $t_r+2T \geq t_{r,v}+2T$ and so at this time, they will have seen all prevotes and precommits seen by $v$ at $t_{r,v}$ and the block $B$ if $v$ broacast it then. By Lemma 5, any honest voter $v'$ has $E_{r-1,v'} \leq B \leq g(V_{r-1,v}$ then.
212 | 
213 | So if the primary broadcast $B$, then $v'$ prevotes for the best chain including $B$. If the primary did not broadcast $B$, then they finalise it. By Lemma 4, it must be that $E_{r-1,v'} \geq B$ and so $E_{r-1,v'}=B$ and so in this case $v'$ also prevots for the best chain including $B$.
214 | 
215 | Since all honest voters prevote $\geq B$, $g(H_r) \geq B$ and so by Lemma 8, all honest particpants finalise $B$ by time $t_r+6T$
216 | 
217 | 
218 | 
219 | 
220 | **Lemma 10** Suppose that $t_r \geq GST+T$ and the primary of round $r$ is honest. 
221 | Let $B$ be the latest block that is ever finalised in rounds  $<r$ (even if no honest paticpant finalises it until after $t_r$). If all honest voters for the prevote in round $r$ agree that the best chain containing $B$ include the same child $B'$ of $B$, then they all finalises some child of $B$ before $t_r+6T$.
222 | 
223 | **proof** By Lemma 4, any honest particapant sees that $E_{r-1} \geq B$ during round $r$. Let $v$ be the pimary of round $r$ and $B''=E_{r-1,v,t_{r,v}}$. If $B'' > B$, then by Lemma 9, all honest validators finalise $B''$ by time $t_r+6T$ which means they finalised a child of $B$. If $B''=B$, then by Lemma 8, all honest voters prevote for the best chain including $B$. By assumption these chains inclue $B'$ and so $g(H_r) \geq B$. By Lemma 8, this means that $B'$ is finalised by time $t_r+6T$.
224 | 
225 | 
226 | 
227 | 
228 | 
229 | 
230 | 
231 | #### Recent Validity
232 | 
233 | **Lemma 11** Suppose that $t_r \geq GST$, the primary of round $r$ is honest and all votes have at most $f$ Byzantine voters. Let $B$ be a block that no prevoter in round $r$ saw as being in the best chain of an ancstor of $B$ at the time they prevoted. Then either all honest validators finalise $B$ before time $t_r+6T$ or no honest validator ever has $g(V_{r,v}) \geq B$ or $E_{r,v} \geq B$.
234 | 
235 | **proof** Let $v'$ be the primary of round $r$ and let $B'=E_{r-1,v',t_{r,v'}}$. If $B' \geq B$, then by Lemma 9, all honest validators finalise $B$ by time $t_r+6T$. If $B' \not\geq B$, then by Lemma 9, no honest validator prevotes $\geq B$ and so no honest validator ever has $g(V_{r,v}) \geq B$.
236 | 
237 | 
238 | **Corollary 2** For $t - 6T > t' \geq GST$, suppose that an honest validator finalises $B$ at time $t$ but that no honest voter has seen $B$ as in the best chain containing some ancestor of $B$ in between times $t'$ and $t$, then at least $(t-t')/6T - 1$ rounds in a row had Byzantine primaries.
239 | 
240 | 
241 | 
242 | 
243 | ## Practicalities
244 | 
245 | ### Alternatives to the last block hash
246 | 
247 | The danger with voting for the last blockhash in the best chain is that maybe no one else will have seen and processed the next block. It would also be nice to make the most of BLS multisig/aggregation, which allows a single signature for many messages/signers than can be checked in time proportional to the number of different mssages signed. 
248 | 
249 | To get round the first alone, it might be better to vote for a block 3/4 along (rounding further) the unfinalised chain , rather than for the head.
250 | 
251 | But the second suggests that maybe we should be including signatures for several of the latest blocks in a chain. We could include that last 2 or 3. We could also do e.g. the the blocks with block numbers with the last 2 multiples of each power of two since the last finalised block, which gives log unfinalis chain length messages but should have many blocks in common.
252 | 
253 | When presented with a vote that includs many blocks, we should interpret them as being for the last block we've seen if any. Then we need to be able to update that vote to a later block when that is seen. This retains monotonicity of a supermajrity for/ it is impossible to have a supermajority for over time.
254 | 
255 | It does not matter if some of the votes are for a block that does not exist as everyone will ignore that part of the vote. But including votes for block that are seen but are not on a chain is an equivocation and is slashable. We need to count such votes as votes for the head of every chain in the vote (as someone might interpret them as for any one of them).
256 | 
257 | Then if we need to BLS aggregate votes that are $\geq B$ for a commit message or query response, it is ok to use any vote that is $\geq B$, not necessarily the vote for the head. This should reduce the number of blockhash signatures, in the optimistic case down to 1.
258 | 
259 | ###  Block production rule
260 | 
261 | If we adopt that rule that block producers should build on the best chain including the finalised block, then if we don't finalise another block this will eventually include some prefix beyond the finalised block, and therefore the protocol is live by Lemma 11.
262 | 
263 | But the issue is that if agreement is much slower than block production, then we might have a prevote for a short chain on the last finalised block, then the best chain does not include that block and we build a long chain that is eventually never finalised. This could be fixed by building on $E_{r-1}$ or $E_r$. But if we do that, and these change very quickly, then we may never come to agreement on the best chain. 
264 | 
265 | So we have two possible chain selection rules for block producers:
266 | 
267 | 1. Build on the best chain including the last finalised block B.
268 | 2. Build on best chain including whichever of $\{E_r,E_{r-1},B\}$ is latest and $\geq B$.
269 | 
270 | 1 is better if finalisation is happening quickly compared to block production and 2 is best if block production is much faster. We could also consider hybrid rules like adopt 1 unless we see that the protocol is stuck or slow, then we switch to 2.
271 | 
272 | ## Why?
273 | 
274 | ### Why do we wait at the end of a round and sometimes before precommitting?
275 | 
276 | If the network  is badly behaved, then these steps may involve waiting an arbitrarily long time. When the network is well behaved (after the GST in our model), we should not be waiting. Indeed there is little point not waiting to recieve 2/3 of voters' votes as we cannot finalise anything without them. But if the gossip network is not perfect, and some messages never arrive, then we may need to implement voters asking other voters for votes from previous rouns in a similar way to the challenge procedure, to avoid deadlock.
277 | 
278 | In exchange for this, we get the property that we do not need to pay attention to votes from before the previous round in order to vote correctly in this one. Without waiting, we could be in a situation where we might have finalised a block in some round $r$, but the network becomes unreliable for many rounds and gets few votes on time, in which case we would need to remember the votes from round $r$ to finalise the block later. 
279 | 
280 | ### Why have a primary?
281 | 
282 | We only need the primary for liveness. We need some form of coordination to defeat the repeated vote splitting attack. The idea behind that attack is that if we are in a situation where almost 2/3 of voters vote for something and the rest vote for another, then the Byzantine voters can control when we see a supermajority for something. If they can carefully time this, they may be able to split the next vote. Without the primary, they could do this for prevotes, getting a supermajority for a block $B$ late, then split precommits so we don't see that it is impossible fo thre to be a supermajority for $B$ until late. If $B$ is not the best block given the last finalised block but $B'$  with the same block number, they could stop either from being finalised like this even if the (unknown) fraction of Byzantine players is small.
283 | 
284 | When the network is well-behaved, an honest primary can defeat this attack by deciding how much we should agree on. We could also use a common coin for the same thing, where poeple would prevote for either the best chain containing $E_{r-1,v}​$ or $g(V_{r-1,v})​$ depending on the common coin. With on-chain voting, it is possible that we could use probabilistic finality of the block production mechanism - that if we don't finalise a block and always build on the best hain containing the last finalised block then not only will the best chain eventually converge, but if a block is behind the head of the best chain, then with positive probability, it will eventually be in the best chain everyone sees.
285 | 
286 | In our setup, having a primary is the simplest option for this.
287 | 


--------------------------------------------------------------------------------
/pdf/grandpa.tex:
--------------------------------------------------------------------------------
  1 | \documentclass{article}
  2 | %
  3 | \usepackage{amsmath} 
  4 | \usepackage{amssymb}
  5 | \usepackage{amsthm}
  6 | \usepackage{color}
  7 | \usepackage{fullpage}
  8 | 
  9 | \usepackage[bookmarks=true, unicode=true, pdftitle={GRANDPA: a blockchain finality gadget}, pdfauthor={Alistair Stewart},pdfkeywords={blockchain finality gadget, consensus, Polkadot, Substrate},pdfborder={0 0 0.5 [1 3]}]{hyperref}
 10 | \usepackage{url}
 11 | \usepackage[numbers]{natbib}
 12 | 
 13 | 
 14 | 
 15 | 
 16 | \usepackage{tabu} %requires array.
 17 | 
 18 | %This should be the last package before \input{Version.tex}
 19 | \PassOptionsToPackage{hyphens}{url}\usepackage{hyperref}
 20 | % "hyperref loads the url package internally. Use \PassOptionsToPackage{hyphens}{url}\usepackage{hyperref} to pass the option to the url package when it is loaded by hyperref. This avoids any package option clashes." Source: <https://tex.stackexchange.com/questions/3033/forcing-linebreaks-in-url/3034#comment44478_3034>.
 21 | % Note also this: "If the \PassOptionsToPackage{hyphens}{url} approach does not work, maybe it's "because you're trying to load the url package with a specific option, but it's being loaded by one of your packages before that with a different set of options. Try loading the url package earlier than the package that requires it. If it's loaded by the document class, try using \RequirePackage[hyphens]{url} before the document class." Source: <https://tex.stackexchange.com/questions/3033/forcing-linebreaks-in-url/3034#comment555944_3034>.
 22 | % For more information on using the hyperref package, refer to e.g. https://en.wikibooks.org/w/index.php?title=LaTeX/Hyperlinks&stable=0#Hyperlink_and_Hypertarget.
 23 | 
 24 | \bibliographystyle{plainnat}
 25 | 
 26 | \newtheorem{theorem}{Theorem}[section]
 27 | \newtheorem{definition}[theorem]{Definition}
 28 | \newtheorem{lemma}[theorem]{Lemma}
 29 | \newtheorem{corollary}[theorem]{Corollary}
 30 | \newtheorem{proposition}[theorem]{Proposition}
 31 | 
 32 | \def\GST{\mathrm{GST}}
 33 | 
 34 | \long\def\com#1{}
 35 | 
 36 | \long\def\xxx#1{{\color{red} {\bf XXX }{\small [#1]}}}
 37 | 
 38 | \begin{document}
 39 | 
 40 | 
 41 | \title{GRANDPA: a Byzantine Finality Gadget}
 42 | \author{Alistair Stewart \\ {\tt stewart.al@gmail.com} \and Eleftherios Kokoris-Kogia \\ {\tt eleftherios.kokoriskogias@epfl.ch}}
 43 | \date{June 30, 2020}
 44 | \maketitle
 45 | 
 46 | \begin{abstract}
 47 | Classic Byzantine fault-tolerant consensus protocols forfeit liveness in the face of asynchrony in order to preserve safety, whereas most deployed blockchain protocols forfeit safety in order to remain live. 
 48 | In this work, we achieve the best of both worlds by proposing a novel abstractions called the \emph{finality gadget}.
 49 | A finality gadget allows for transactions to always optimistically commit but informs the clients that these transactions might be unsafe. As a result, a blockchain can execute transactions optimistically and only commit them after they have been sufficiently and provably audited.
 50 | In this work, we formally model the finality gadget abstraction, prove that it is impossible to solve it deterministically in full asynchrony (even though it is stronger than consensus) and provide a partially synchronous protocol which is currently securing a major blockchain. This way we show that the protocol designer can decouple safety and liveness in order to speed up recovery from failures. We believe that there can be other types of finality gadgets that provide weaker safety (e.g., probabilistic) in order to gain more efficiency and this can depend on the probability that the network is not in synchrony.
 51 | \end{abstract}
 52 | \section{Introduction}
 53 | 
 54 | 
 55 | Bitcoin~\cite{nakamoto08bitcoin} and its descendants~\cite{wood14ethereum,sasson2014zerocash} are cryptocurrencies that provide 
 56 | secure automated value exchange without the need of a central managing authority.
 57 | Instead a decentralized consensus protocol maintains a distributed public ledger
 58 | known as the \textit{blockchain}. To be able to rely on public ledger one needs to know that it has reached consensus on a certain block, i.e., when a block will not be reverted anymore, which we refer to as reaching finality.  
 59 | One of the challenges of Nakomoto-like consensus protocols is that they only satisfy eventual consensus, which only guarantees that an ever growing prefix of the chain will be agreed upon by all participants forever onward. The eventual consensus process generally takes tens of minutes and it only gives probabilistic guarantees (for a certain block number at a certain point in time). 
 60 | 
 61 | Unfortunately these guarantees only hold if the underlying network is well-connected and the client is able to find an uncensored source of information, two assumptions that
 62 | do not hold in adversarial environments~\cite{apostolaki16hijacking, gervais15tampering, heilman15eclipse}.
 63 | The underlying problem which enables these attacks is that first generation blockchain protocols do not consider finality (i.e., when will a block never be reverted) as a first class property, prioritising liveness instead.
 64 | 
 65 | An alternative to probabilistic finality is having \emph{provable finality} where anyone can be convinced of the finality of a block, regardless of being a consensus participants or actively following the network.% Provable fnality can be reached for example, a signed statement by a set of authorities, the set of whom can be tracked, that the block is final.
 66 | %This is useful to prove what happened to light clients, who do not have the full chain or are not actively listening to the network, and to communicate with other chains, possibly as part of a scalability solution, where not anyone receives or stores all the data in the system.
 67 | New generation protocols~\cite{kokoris16enhancing,decker16bitcoin,pass16hybrid} propose the complete opposite. They propose every block to be finalized one by one and to forfeit liveness when finality is not readily achievable. This gives provable finality immediately.% However this is slow if we have a large set of participants in the Byzantine agreement.
 68 | Unfortunately, these types of protocols inherit the shortcoming of classic consensus protocol on losing performance when many nodes are required to participate. Hence, they need to put a limit on the number of consensus participants which might lead to centralization. 
 69 | 
 70 | %Secondly, tying consensus with liveness of the chain means that there is no optimism and blocks are committed only when all verification information are available. As a result, they lose the ability to make some (unsafe) progress when the network is unstable, which can help speedup the recovery process when the network heals. 
 71 | 
 72 | 
 73 | In this work we show the that the middle ground also merits exploration. The approach that we will take is similar to the approach that Ethereum plans to take with Casper the Friendly Finality Gadget (Casper FFG)[2].
 74 | We introduces and formalize the idea of lazy finality which is encapsulated in the abstraction of a \emph{finality gadget.} Separating the liveness of the consensus protocol from the finality of the blocks. This approach has three concrete benefits for the overlying blockchain protocol. 
 75 | 
 76 | %We will use a block production mechanism and chain selection rule that give eventual consensus and then add a finality gadget, a protocol that finalises blocks that the participants already agree on, to get provable finality.
 77 | 
 78 | A first benefit is that since consensus is not tied to liveness of the chain we can have optimistic execution.  As a result, the chain can grow before it is certain that blocks are valid. Later on, we can finalize blocks when we are sure they are correct i.e., all verification information is available. 
 79 | A second benefit is that we can make some (unsafe) progress when the network is unstable. This enables a fast recovery process when the network heals. Similarly, we can make progress in chain growth even when finlzation is slow, e.g., when we have many particiapnts thus promoting decentenralization. 
 80 | 
 81 | %First, not every node that is competing to mine a block needs to be part of the finality gadget, hence we can get the \textit{best of both worlds} (i.e., full inclusive decentralization and scalable fast finality). 
 82 | 
 83 | %Second, we can reorganize the chain for blocks that have not yet been finalized \xxx{Al put a sentence on why this is a cool property.}. 
 84 | 
 85 | Third, a finality gadget can be deployed gradually and light clients can choose to consult it or follow the longest chain rule and ignore it, enabling light client heterogeneity. 
 86 | The light client that trust the gadget do not need to have the full chain or actively listen to the network. 
 87 | This can in turn enable scalability~\cite{avarikioti19divide} in an ecosystem of multiple chains (weather sharding~\cite{kokoris17omniledger,al18chainspace,androulaki18channels} or heterogeneous~\cite{zamyatin19sok}), where no single party receives or stores all the data in the system. 
 88 | 
 89 | %Finally, the finality gadget protocol is simpler than consensus protocols since they do not require a complex and costly view change phase.
 90 | 
 91 | In short, we fromalize the abstraction of a \emph{finality gadget} that runs along any block production mechanism (e.g., Nakamoto consensus) providing provable finality guarantees. We show that it is impossible to satisfy its properties with a deterministic asynchronous protocol. To circumvent this impossibility result, we introduce the GRANDPA finality gadget that works in a partially synchronous network model, in the presence of up to $1/3$ Byzantine actors. 
 92 | %To this end we formalise the finality gadget problem and 
 93 |  
 94 | 
 95 | The combination of GRANDPA with a classic block production mechanism like GHOST~\cite{lewenberg15inclusive} results in the existing deployment of the polkadot network \footnote{\url{https://polkadot.network}} which provides fast finality under good network conditions and protects the clients without compromising the liveness when under attack. The implementation of GRANDPA is available on github \footnote{See \url{https://github.com/paritytech/finality-grandpa} and \url{https://github.com/paritytech/substrate/tree/master/client/finality-grandpa}}.
 96 | 
 97 | 
 98 | In summary we make the following contributions:
 99 | \begin{itemize}
100 | 
101 | \item Introduce the idea of lazy finality and instantiate it through a finality gadget abstraction
102 | 
103 | \item Prove that BFG is impossible in asynchrony and present GRANDPA
104 | 
105 | \end{itemize}
106 | 
107 | 
108 | %The approach that we will take is similar to the approach that Ethereum plans to take with Casper the Friendly Finality Gadget (Casper FFG)\cite{CasperFFG}, which combines these approaches.
109 | %We will use a block production mechanism and chain selection rule that give eventual consensus and then add a finality gadget, a protocol that finalises blocks that the participants already agree on, to get provable finality.
110 | 
111 | %We present a finality gadget that works in a partially synchronous network model, GRANDPA, that works in the presence f up to $1/3$ Byzantine actors as well as an asynchronous finality gadget, that can cope with $1/5$ Byzantine actors. 
112 | 
113 | 
114 | 
115 | \section{Model, Definitions, and Impossibilities}
116 | 
117 | 
118 | We want to formalise the notion of finality gadget to be a sub-protocol that can be deployed along any protocol with
119 | eventual consensus and probabilistic finality and enhancing such protocol with provable finality.
120 | To achieve this, we need to incorporate into the classic definition of Byzantine agreement 
121 | the fact that we additionally have access to a protocol that would achieve eventual consensus if we did not affect it.
122 | 
123 | \subsection{Byzantine Agreement with a Consistency Oracle}
124 | Consider a typical definition of a multi-values Byzantine agreement: 
125 | We have a set of participants $V$, the majority of whom obey the protocol, but a constant fraction may be Byzantine, meaning they behave arbitrarily, e.g. provide false or inconsistent information or randomly go offline when they ought to be online.
126 | 
127 | \begin{definition} A protocol for {\em multi-valued Byzantine agreement} has a set of values $S$ and a set of voters $V$, a constant fraction of which may be Byzantine, for which each voter $v \in V$ starts with an initial value $s_v \in S$ and, in the end, decides a final value $f_v \in S$ such that the following holds:
128 | 
129 | \begin{itemize}
130 | \item {\bf Agreement}: All honest voters decide the same value for $f_v$
131 | \item {\bf Termination}: All honest voters eventually decide a value
132 | \item {\bf Validity}: If all honest voters have the same initial value, then they all decide that value
133 | \end{itemize}
134 | 
135 | \end{definition}
136 | 
137 | We can change this definition to assume that instead of having an initial value, all voters have access to an external protocol, an oracle for values, that achieves eventual consensus in that it returns the same value to all voters when called after some unknown time.
138 | 
139 | \begin{definition}
140 | We say an oracle $A$ in a protocol is {\em eventually consistent} if it returns the same value to all participants after some unspecified time.
141 | \end{definition}
142 | 
143 | 
144 | \begin{definition} A protocol for the {\em multi-valued Byzantine finality gadget problem} has a set of values $S$, a set of voters $V$, a constant fraction of which may be Byzantine, for which each voter $v \in V$ has access to an eventually consistent oracle $A$ and, in the end, each voter decides a final value $f_v \in S$ such that the following holds:
145 | 
146 | \begin{itemize}
147 | \item {\bf Agreement:} All honest voters decide the same value for $f_v$
148 | \item {\bf Termination:} All honest voters eventually decide a value
149 | \item {\bf Validity:} All honest voters decide a value that $A$ returned to some honest voter sometime.
150 | \end{itemize}
151 | 
152 | \end{definition}
153 | 
154 | \paragraph{Impossibility of Deterministic Agreement with an Oracle.}\label{ssec:impossibility}
155 | %\xxx{Al can you extend this?}
156 | For the binary case, i.e. when $|S|=2$, the Byzantine finality gadget problem is reducible to Byzantine agreement. This does not hold for $|S| > 2$, because the definition of validity is stronger in our protocol. Note that it is impossible for multi-valued Byzantine agreement to make the validity condition require that we decide an initial value of some honest voter and tolerate more than a $1/|S|$ fraction of faults, since we may have a $1/|S|$ fraction of voters reporting each initial value and Byzantine voters can act honestly enough not to be detectable. For finality gadgets, this stronger validity condition is possible. A natural question is then weather the celebrated FLP~\cite{flp} impossibility holds for our stronger requirements.
157 | Next, we show that an asynchronous, deterministic binary finality gadget is impossible, even with one fault. 
158 | This means that the extra information voters have here, that $A$ will eventually agree for all voters, is not enough to make this possible.
159 | 
160 | 
161 | \paragraph{Proof:}
162 | %\xxx{TODO:Al}
163 | The asynchronous binary fault tolerant agreement problem is as follows:
164 | 
165 | We have  number of voters which each have an initial $v_i$ in $\{0,1\}$
166 | 
167 | We may have one or more faulty nodes, which here means going offline at some point. Nodes have asynchronous communication - so any message arrives but we have no guarantee when it will.
168 | The goal is to have all non-faulty nodes output the same $v$, which must be $0$ if all inputs $v_i$ are $0$ and $1$ if all are $1$.
169 | 
170 | Fischer, Lynch and Paterson\cite{flp} showed that this is impossible if there is one faulty node.
171 | 
172 | The binary fault-safe finality gadget problem is similar, except now there is an oracle $A$ that any node can call at any time with the following properties:
173 | 
174 | either $A$ always outputs $x$ in $\{0,1\}$ to all nodes at all times
175 | or else there is an $x$ in $\{0,1\}$ and
176 | for each node $i$, there is a $T_i$ such that when $i$ calls $A$ before $T_i$. it gives $x$ but if it calls $A$ after $T_i$, it returns not $x$ .
177 | 
178 | and we want that if A never switches, then all non-faulty nodes output x. If A does switch then all non-faulty nodes should output the same thing, but it can be 0 or 1. 
179 | 
180 | Then this is also impossible, even for one faulty node, which just goes offline. Note that this generalises Byzantine agreement, since if we could each node $i$ could call $A$ once at the start and use the output as $v_i$. (For the multi-valued case, we will define the problem so that this reduction does not hold.)
181 | 
182 | 
183 | \begin{proof}[Proof sketch] We follow the notation of  \cite{flp} and assume for a contradiction that we use a correct protocol. 
184 | Let $r$ be a run of the protocol where $A$ gives $0$ all the time.
185 | Then by correctness $r$ decides $0$. Now we consider what can happen when $A$ switches to $1$ after each configuration in $r$. If it switches to $1$ at the start, then the protocol decides $1$.
186 | If we switch to $1$ when all node have already decided $0$, then we decide $0$.
187 | 
188 | We claim that some configuration in the run $r$, where there are two runs from it where $A$ is always $1$ that decide $0$ and $1$. We call such states $1$-bivalent.
189 | To see this, assume for a contradiction that $r$ contains no such configurations. Then there are successive configurations $C$,$C'$ such that if $A$ return $1$ in the future from $C$ then we always decide $0$ but from $C'$, we always decide $1$.
190 | Let events be $(p,m,x)$ where node (processor/voter) $p$ receives message $m$ (which may be null) and executes some code where any calls to A return $x$ in $\{0,1\}$, then sends some messages. 
191 | Then there is some event $(p,m,0)$ that when applied to $C$ gives $C'$. Now suppose that $p$ goes offline at $C$, then if $A$ always returns $1$ afterwards, then we still decide $1$. Thus there is a run $r'$ that starts at $C$ where $p$ takes no steps, $A$ always returns $1$ and all other nodes still output $1$.
192 | But since $p$ takes no steps in $r'$, we can apply $r'$ after $(p, m, 0)$ and so we have that $C'$ has a run where $A$ always returns $1$ but decides $1$, which is a contradiction.
193 | 
194 | Now let $C$ be a $1$-bivalent configuration. We can follow the FLP proof to show that there is a run from $C$ for which $A$ always returns $1$, all messages are delivered but all configurations are 1-bivalent and so the protocol never decides. This completes the proof by contradiction that there is no correct protocol.
195 | \end{proof}
196 | 
197 | 
198 | 
199 | \subsection{Definition of a Finality Gadget}
200 | 
201 | In this section we show how to extend the one-shot agreement to agreeing on a chain of blocks. One difficulty in formalising the problem is that the block production mechanism cannot be entirely separate from the finality gadget. In order to finalise new blocks, we must first build on the chain we have already finalised. So at a minimum, the block production mechanism needs to recognise which blocks the finality gadget has finalised. We will also allow the block production mechanism to interact with the state of the finality gadget in other ways.
202 | 
203 | We want the finality gadget to work with the most general block production mechanisms as possible. Thus we need a condition that combines the property of eventual consensus and this requirement to build on the last finalised block, but is otherwise not too restrictive.
204 | We assume a kind of conditional eventual consensus.
205 | If we keep building on our last finalised block $B$ and don't finalise any new blocks, then eventually we have consensus on a longer chain than just $B$, which the finality gadget can use to finalise another block.
206 | We also want a protocol that does not terminate, but instead keeps on finalising more blocks. 
207 | 
208 | We assume that there is a block production protocol $P$ that runs at the same time as the finality gadget protocol $G$. Actors who are participants in both protocols may behave differently in $P$ depending on what happened in $G$.
209 | However in the reverse direction, the only way that an honest voter $v$'s behaviour in $G$ is affected by $P$ is through a voting rule, a function $A(v,s_v,B)$ that depends on $v$ and its state $s_v$ and takes a block $B$ and returns a block $B'$ at the head of a chain including $B$.
210 | 
211 | We say that the system $G$, $P$, and $A$ achieves {\em conditional eventual consensus}, if $G$ has finalised a block $B$, then eventually, either $G$ will finalise some descendant of $B$ or else all the chains with head $A_{v,s_v}(B)$ for all voters $v$ at all future states $s_v$ will contain the same descendant $B'$ of $B$.
212 | 
213 | \begin{definition} \label{def:finality-gadget} 
214 | Let $F$ be a protocol  with a set of voters $V$, a constant fraction of which may be Byzantine.
215 | We say that $F$ solves {\em blockchain Byzantine finality gadget problem} if for every block production protocol $P$ and voting rule $A$ we have the following
216 | 
217 | %A protocol for the blockchain Byzantine finality gadget problem has , each of whom has access to an oracle $A$ for the best chain given the last finalised block with the property that, as long as no new block is finalised, it achieves eventual consensus on some child of the last finalised block such that the following holds:
218 | 
219 | \begin{itemize}
220 | \item{\bf Safety:} All honest voters finalise the same block at each block number.
221 | \item{\bf Liveness:} If the system $F, G, A$ achieves conditional eventual consensus, then all honest voters keep finalising blocks.
222 | \item{\bf Validity:} If an honest voter finalises a block $B$ then that block was seen in the best chain observed by some honest voter containing some previously finalised ancestor of $B$,
223 | \end{itemize}
224 | 
225 | \end{definition}
226 | 
227 | %\xxx{Lef I do not understand the paragraph below, clarify.}
228 | 
229 | As an example, we could assume $F$ uses proof of work to build on the longest chain and includes the last block $G$ finalised. Then we take $A(v,s_v,B)$ as being the longest chain which includes $B$ and which $v$ sees in state $s_v$. It is well-known \cite{nakamoto08bitcoin} that longest chain with proof of work achieves eventual consensus under the right assumptions and similar arguments show that in this case we have conditional eventual consensus.
230 | As long as we do not change the chain we are building on by finalising another block, we will eventually agree on some prefix longer than the last finalised block.
231 | Thus, any finality gadget that satisfies Definition \ref{def:finality-gadget} will work in this system so that all honest voters finalise an increasingly long common chain.
232 | Thanks to the abstraction above, we can switch $F$ for one of many possible alternative consensus algorithms and $G$ will still work.
233 | 
234 | 
235 | \com{
236 | 
237 | \subsection{Our results}
238 | 
239 | 
240 | 
241 | \subsection{Related Work}
242 | 
243 | \subsubsection{Comparison with Casper}
244 | 
245 | The concept of finality gadget was introduced in Casper the friendly finality gadget and this remains the finality gadget which is most similar to ours. So it makes sense to compare these. However first, we should mention the other protocols that are also called Casper.
246 | 
247 | The first Casper was Casper TFG. Casper CBC\cite{CasperCBC} gives a recent and clearly specified version of this protocol. It's fork choice rule uses the GHOST selection rule on votes.
248 | In Casper TFG, votes are blocks, but they are counted by participants (proposers and validators) like votes, which differs from how GHOST would be used with proof of work. It also has a flexible way of subjectively finalising blocks based on graphs of votes. 
249 | 
250 | In Casper FFG\cite{CasperFFG}, validators vote on links between checkpoints, which occur at block numbers that are multiples of, say, 50. If there are 2/3 votes for one block at consecutive checkpoints, then we can finalise a chain of blocks up to the first checkpoint.
251 | 
252 | Epochless Casper, 
253 | 
254 | Casper...
255 | 
256 | There are two main differences between Casper FFG and GRANDPA. One is that in GRANDPA, different voters can cast votes simultaneously for blocks at different heights. This is achieved by borrowing the concept of GHOST on votes from Casper TFG and applying it in a more traditional Byzantine agreement protocol.
257 | 
258 | The other main difference is how the finality gadget affects the fork-choice rule of the underlying block production mechanism. In GRANDPA, by default we will assume that this is only affected by having to include any finalised blocks. 
259 | Casper FFG \cite{CasperFFG} does not specify a fork-choice rule, but it requires that we build on justified blocks for liveness. Later specifications of Casper use the GHOST rule on votes for fork-choice.
260 | 
261 | Only depending on finalised blocks gives a clearer separation between the block production mechanism and finality gadget. It may therefore be easier to adapt to other types of protocol that achieve eventual consensus—and there have been many diverse protocols that do this developed in the last few years.
262 | It also makes it far easier to prove liveness properties.
263 | If the finality gadget has not finalised anything and so does not interfere, then the underlying mechanism should reach eventual consensus, which should be enough for the finality gadget to finalise whatever we have consensus on.
264 | 
265 | On the other hand, while building on the longest chain in the absence of a finality gadget to maximize block rewards may be rational if everyone else does, this is not always the case for building on the longest chain including the last finalised block.
266 | This is because it may be likely that a different chain is going to be finalised, in which case the rational thing to do might be to build on that. The GHOST on votes fork choice rule of ? and ? may be more rational.
267 | It is not clear that it is, nor is it clear how to prove liveness for such a rule. Further research may be needed to show that there is a fork choice rule which is rational and leads to liveness for the finality gadget. 
268 | 
269 | 
270 | 
271 | }
272 | \subsection{Preliminaries} \label{sec:prelims}
273 | 
274 | \paragraph{Network model}: We will be using the partially synchronous network model introduced by \cite{DLS} and in particular the gossip network variant used in \cite{Tendermint}.
275 | We assume that any message sent or received by an honest participant reaches all honest participants within time $T$, but possibly only after some Global Synchronisation Time $\GST$.
276 | Concretely, any message sent or received by some honest participant at time $t$ is received by all honest participants by time $\GST+T$ at the latest.
277 | 
278 | 
279 | \paragraph{Voters:} \com{We will want to change the set of participants who actively agree sometimes. 
280 | To model this, we have a large set of participants who follow messages.}
281 | For each voting step, there is a set of $n$ voters.
282 | We will frequently need to assume that for each such step, at most  $f < n/3$ voters are Byzantine.
283 | We need $n-f$ of voters to agree on finality. Whether or not block producers ever vote, they will need to be participants who track the state of the protocol.
284 | 
285 | \paragraph{Votes:}A vote is a block hash, together with some metadata such as round number and the type of vote, such as {\em prevote} or {\em precommit}, all signed with a voter's private key.
286 | 
287 | \paragraph{Rounds:}Each participant has their own idea of what is the current round number. Every prevote and precommit has an associated round number. Honest voters only vote once (for each type of vote) in each round and do not vote in earlier rounds after later ones.
288 | Participants need to keep track of which block they see as currently being the latest finalised block and an estimate of which block could have been finalised in the last round. 
289 | 
290 | 
291 | For block $B$, we write $\mathrm{chain}(B)$ for the chain whose head is $B$. The block number, $n(B)$ of a block $B$ is the length of $\mathrm{chain}(B)$.
292 | For blocks $B'$ and $B$, we say $B$ is later than $B'$ if it has a higher block number.
293 | We write $B > B'$ or that $B$ is descendant of $B'$ for $B$, $B'$ appearing in the same blockchain with $B'$ later i.e. $B' \in \mathrm{chain}(B)$ with $n(B) > n(B')$.
294 | $B \geq B'$ and $B \leq B'$ are similar except allowing $B = B'$.
295 | We write $B \sim B'$ or $B$ and $B'$ are on the same chain if $B<B'$, $B=B'$ or $B> B'$; and $B \nsim B'$ or $B$ and $B'$ are not on the same chain if there is no such chain.
296 | 
297 | Blocks are ordered as a tree with the genesis block as root. So any two blocks have a common ancestor but two blocks not on the same chain do not have a common descendant.
298 | A vote $v$ for a block $B$ by a voter $V$ is a message signed by $V$ containing the blockhash of $B$ and meta-information like the round numbers and the type of vote. 
299 | 
300 | A voter equivocates in a set of votes $S$ if they have cast multiple different votes in $S$. We call a set $S$ of votes safe if the number of voters who equivocate in $S$ is at most $f$. We say that $S$ has a supermajority for a block $B$ if the set of voters who either have a vote for blocks $\geq B$ or equivocate in $S$ has size at least $(n+f+1)/2$.  We count equivocations as votes for everything so that observing a vote is monotonic, meaning that if $S \subset T$ then if $S$ has a supermajority for $B$ so does $T$, while being able to ignore yet more equivocating votes from an equivocating voter.
301 | 
302 | For our finality gadget (GRANDPA) we use the ghost~\cite{lewenberg15inclusive} eventual consensus algorithm as $F$.
303 | The $2/3$-GHOST function $g(S)$ takes a set $S$ of votes and returns the block $B$ with highest block number such that $S$ has a supermajority for $B$.
304 | If there is no such block, then it returns `nil`. \com{(if $f \neq \lfloor (n-1)/3 \rfloor$, then this is a misnomer and we may change the name of the function accordingly.)}
305 | Note that, if $S$ is safe, then we can compute $g(S)$ by starting at the genesis block and iteratively looking for a child of our current block with a supermajority, which must be unique if it exists. Thus we have:
306 | \begin{lemma} \label{lem:ghost-monotonicity}
307 | Let $T$ be a safe set of votes. Then
308 | \begin{enumerate}
309 | \item The above definition uniquely defines $g(T)$
310 | \item If $S \subseteq T$ has $g(S) \neq$ nil, then $g(S) \leq g(T)$.
311 | \item If $S_i \subseteq T$ for $1 \leq i \leq n$ then all non-nil $g(S_i)$ are on a single chain with head $g(T)$.
312 | \end{enumerate}
313 | 
314 | \end{lemma}
315 | 
316 | Note that we can easily update $g(S)$ to $g(S \cup \{v\})$, by checking if any child of $g(S)$ now has a supermajority.
317 | The third rule tells us that even if participants see different subsets of the votes cast in a given voting round, this rule may give them different blocks but all such blocks are in the same chain under this assumption. 
318 | 
319 | Next, we define a notion of possibility to have a supermajority which says that if the set of all votes in a vote $T$ is safe and some participant observes a subset $S \subseteq T$ that has a supermajority for a block $B$ then all participants who see some other subset $S' \subseteq T$ still see that it is possible for $S$ to have a supermajority for $B$. We need a definition that extends to unsafe sets.
320 | We say that it is {\em impossible} for a set $S$ to have a supermajority for $B$ if at least $(n+f+1)/2$ voters either vote for a block $\not \geq B$ or equivocate in $S$. Otherwise it is {\em possible} for $S$ to have a supermajority for $B$.
321 | 
322 | Note that if $S$ is safe, it is possible for $S$ to have a supermajority for $B$ if and only if there is a safe $T \supseteq S$ that has a supermajority for $B$, which can be constructed by adding a vote from $B$ for all voters without votes in $S$ and enough voters who already have votes in $S$ to bring the number of equivocations up to $f$.
323 | 
324 | We say that it is {\em impossible} for any child of $B$ to have a supermajority in $S$ if $S$ has votes from at least $2f+1$ voters and it is impossible for $S$ to have a supermajority for each child of $B$ appearing on the chain of any vote in $S$.
325 | Again, provided $S$ is safe, this holds if and only if for any possible child of $B$, there is no safe $T \subseteq S$ that has a supermajority for that child.
326 | \com{
327 | Note that it is possible for an unsafe $S$ to both have a supermajority for $S$ and for it to be impossible to have such a supermajority under these definitions, as we regard such sets as impossible anyway.
328 | }
329 | \begin{lemma} \label{lem:impossible}
330 | \begin{itemize}
331 | \item[(i)] If $B' \geq B$ and it is impossible for $S$ to have a supermajority for $B$, then it is impossible for $S$ to have a supermajority for $B'$.
332 | \item[(ii)] If $S \subseteq T$ and it is impossible for $S$ to have a supermajority for $B$, then it is impossible for $T$ to have a supermajority for $B$.
333 | \item[(iii)] If $g(S)$ exists and $B \nsim g(S)$ then it is impossible for $S$ to have a supermajority for $B$.
334 | \end{itemize}
335 | \end{lemma}
336 | 
337 | 
338 | \section{Finality Gadget Protocols} \label{sec:finality}
339 | 
340 | \com{
341 | To discover up with a solution to the blockchain Byzantine finality gadget problem, we will typically look at various Byzantine agreement protocols and use those to find protocols for the multi-valued Byzantine finality gadget problem. 
342 | Agreement protocols with appropriate properties can be used to find protocols for the blockchain Byzantine finality gadget problem by considering running them in parallel at every block number.
343 | If the one block protocol has the right properties then they will agree on blocks consistently, so if we finalise a block then we also finalise its ancestors and we can come up with a succinct protocol.
344 | 
345 | For example, suppose we have a one block protocol that calls for a vote on blocks which requires a participant to observe a supermajority, say votes from  $2/3$ of voters, for some block, or else the participant observes that the vote is undecided. Now imagine running this vote in parallel for every block number and have any honest voter vote for blocks from a particular chain.
346 | Byzantine voters may vote more than once, but if we count a vote for a block as a vote for each ancestor of the block in the vote for the instance of the one block protocol with its number, then Byzantine voters must also vote for chains, though they can vote for multiple chains.
347 | If we do this, then we see that if a block has a supermajority in a vote, then so does all its ancestors in their votes. Thus the blocks with a supermajority form a chain.
348 | Furthermore, if only $1/3$ of voters equivocate then if a participant sees a subset of the votes for chains, then they must see a prefix of the chain of blocks for which all the votes have supermajorities. Intuitively, the protocol can agree on the prefix that $2/3$ of voters agree on using this. 
349 | 
350 | To ensure safety, each participant maintains an estimate $E_r$ of the last block that could have been finalised in a round $r$. This has the property that in future rounds it overestimates the block that could have been finalised so that in round $r$, the chain with head $E_{r-1}$ contains all blocks that could have been finalised.
351 | Any honest voter only votes in round $r$ for chains containing their estimate $E_{r-1}$ and this guarantees that any block that could have been finalised in round $r-1$ will be finalised in round $r$.
352 | }
353 | 
354 | In order to find a solution to the finality gadget protocol we look in
355 | consensus protocols that solve the stronger problem as described in the previous section. The key idea for our solution is to inherit the safety properties of a consensus protocol, but use the underlying blockchain as the driving force of liveness. This results in a protocol which does not stop when for example the network is split. 
356 | Instead, only the finalization stops, but the blocks keep getting created and propagated to everyone.
357 | This means that when the conditions are safe again, the finality gadget only needs to finalize the head of the chain\footnote{Which the oracle will return quickly to a supermajority of miners.},
358 | instead of having to transmit and run consensus on every block. 
359 | %In Figure~\ref{fig:finality}, we analyze the differences between classic blockchain protocols~\cite{nakamoto08bitcoin,wood14ethereum}, finality gadget, and hybrid consensus solutions~\cite{kokoris16enhancing,gilad17algorand}
360 | %\xxx{Experiment: Catchup 100s of blocks Hotstuff vs GRANDPA}.
361 | 
362 | 
363 | 
364 | 
365 | 
366 | \subsection{The GRANDPA Protocol}\label{sec:grandpa}
367 | In this section, we give our solution to the Byzantine finality gadget problem, GRANDPA.  Our finality gadget works the partially synchronous setting, we also provide a fully asynchronous solution in Appendix~\ref{app:async}.
368 | 
369 | GRANDPA works in rounds, each round has a set of $3f+1$ eligible voters, $2f+1$ of which are assumed honest. Furthermore, we assume that each round has a participant designated as primary and all participants agree on the voter sets and primary. We will can either choose the primary pseudorandomly from or rotate through the voter set.
370 | On a high-level, each round consists of a double-echo protocol after which every party waits in order to detect whether we can finalize a block in this round (this block does not need to be the immediate ancestor of the last finalized block, it might be far ahead from the last finalized block). If the round is unsuccessful, the parties simply move on to the next round with a new primary. When a good primary is selected, the oracle is consistent (returns the same value to all honest parties),
371 | and the network is in synchrony (after $\GST$), then a new block will be finalized and it will transitively finalized all its ancestors.
372 | 
373 | More specifically, we let $V_{r,v}$ and $C_{r,v}$ be the sets of prevotes and precommits respectively received by $v$ from round $r$ at the current time.
374 | 
375 | We define $E_{r,v}$ to be $v$'s estimate of what might have been finalised in round $r$, given by the last block in the chain with head $g(V_{r,v})$ for which it is possible for $C_{r,r}$ to have a supermajority. Next we define a condition which will allow us to safely conclude that $E_{r,v} \geq B$ for all $B$ that might be finalised in round $r$:
376 | If either $E_{r,v} < g(V_{r,v})$ or it is impossible for $C_{r,v}$ to have a supermajority for any children of $g(V_{r,v})$, then we say that {\em $v$ sees that round $r$ is completable}.  $E_{0,v}$ is the genesis block, assuming we start at $r=1$.  
377 | 
378 | In other words, a round $r$ is completable when our estimate chain $E_{r,v}$ contains everything that could have been finalised in round $r$, which makes it possible to begin the next round $r+1$.
379 | 
380 | We have a time bound $T$ that after $\GST$ suffices for all honest participants to communicate with each other.
381 | Inside a round, the properties both of $E_{r,v}$ having a supermajority, meaning $E_{r,v} < g(V_{r,v})$, as well as of it being impossible to have a supermajority for some given block are monotone, so the property of being completable is monotone as well.
382 | We therefore expect that, if anyone sees a round is completable, then everyone will see this within time $T$. Leaving a gap of $2T$ between steps is then enough to ensure that every party receives all honest votes before continuing.
383 | 
384 | 
385 | \paragraph{Protocol Description.}
386 | In round $r$ an honest participant $v$ does the following:
387 | 
388 | \noindent \fbox{\parbox{6.3in}{
389 | 
390 | \begin{enumerate}
391 | \item A voter $v$ can start round $r > 1$ when round $r-1$ is completable and $v$ has cast votes in all previous rounds where they are a voter. Let $t_{r,v}$ be the time $v$ starts round $r$.
392 | 
393 | \item At time $t_{r,v}$, if $v$ is the primary of this round and has not finalised $E_{r-1,v}$ then they broadcast $E_{r-1,v}$. If they have finalised it, they can broadcast $E_{r-1,v}$ anyway (but do not need to).
394 | 
395 | \item If $v$ is a voter for the prevote of round $r$, $v$ waits until either it is at least time $t_{r,v}+2T$ or round $r$ is completable, then broadcasts a prevote.
396 | They prevote for the head of the best chain containing $E_{r-1,v}$ unless we received a block $B$ from the primary and $g(V_{r-1,v}) \geq B > E_{r-1,v}$, in which case they use the best chain containing $B$ instead.
397 | 
398 | \item If $v$ is a voter for the precommit step in round $r$, then they wait until $g(V_{r,v}) \geq E_{r-1,v}$ and one of the following  conditions holds
399 | \begin{itemize}
400 | \item[(i)] it is at least time $t_{r,v}+4T$, 
401 | \item[(ii)] round $r$ is completable or
402 | \item[(iii)] it is impossible for $V_{r,v}$ to have a supermajority for any child of $g(V_{r,v})$,
403 | \end{itemize}
404 | and then broadcasts a precommit for $g(V_{r,v})$ {\em( (iii) is optional, we can get away with just (i) and (ii))}.
405 | 
406 | \end{enumerate}
407 | 
408 | }}
409 | 
410 | Note that $C_{r,v}$ and $V_{r,v}$ may change with time and also that $E_{r-1,v}$, which is a function of $V_{r-1,v}$ and $C_{r-1,v}$, can also change with time if $v$ sees more votes from the previous round.
411 | 
412 | \paragraph{Finalisation.}
413 | 
414 | If, for some round $r$, at any point after the precommit step of round $r$, we have that $B=g(C_{r,v})$ is later than our last finalised block and $V_{r,v}$ has a supermajority, then we finalise $B$.
415 | We may also send a commit message for $B$ that consists of $B$ and a set of precommits for blocks $\geq B$ (ideally for $B$ itself if possible see "Alternatives to the last blockhash" below). 
416 | 
417 | To avoid spam, we only send commit messages for $B$ if we have not receive any valid commit messages for $B$ and its descendants and we wait some time chosen uniformly at random from $[0,1]$ seconds or so before broadcasting.
418 | If we receive a valid commit message for $B$ for round $r$, then it contains enough precommits to finalise $B$ itself if we haven't already done so, so we'll finalise $B$ as long as we are past the precommit step of round $r$.
419 | 
420 | 
421 | \com{
422 | \subsection{Discussion}
423 | 
424 | \paragraph{Wait at the end of a round before precommitting.}
425 | 
426 | If the network  is badly behaved, then these steps may involve waiting an arbitrarily long time. When the network is well behaved (after the $\GST$ in our model), we should not be waiting. Indeed there is little point not waiting to receive $2f+1$ of voters' votes as we cannot finalise anything without them.
427 | But if the gossip network is not perfect and some messages never arrive, then we may need to make voters asking other voters for votes from previous rounds.\com{ in a similar way to the challenge procedure, to avoid deadlock.}
428 | 
429 | In exchange for our design choice of waiting, we get the property that we do not need to pay attention to votes from before the previous round in order to vote correctly in this one. Without waiting, we could be in a situation where we might have finalised a block in some round r, but the network becomes unreliable for many rounds and gets few votes on time, in which case we need to remember the votes from round r to finalise the block later. 
430 | 
431 | \subsubsection{Using a Primary}
432 | 
433 | We only need the primary for liveness. 
434 | We need some form of coordination to defeat the repeated vote splitting attack. The idea behind that attack is that if we are in a situation where almost 2/3 of voters vote for something an the rest vote for another, then the Byzantine voters can control when we see a supermajority for something. If they can carefully time this, they may be able to split the next vote. 
435 | Without the primary, they could do this for prevotes, getting a supermajority for a block $B$ late, then split precommiher from being finalised like this even if the (unknown) fraction of Byzantine players is small.
436 | 
437 | When the network is well-behaved, an honest primary can defeat this attack by deciding how much we should agree on. We could also use a common coin for the same thing, where people would prevote for either the best chain containing $E_{r-1,v}$ or $g(V_{r-1,v})$ depending on the common coin.
438 | With on-chain voting, it is possible that we could use probabilistic finality of the block production mechanism - that if we don't finalise a block and always build on the best chain containing the last finalised block then not only will the best chain eventually converge, but if a block is behind the head of the best chain, then with positive probability, it will eventually be in the best chain everyone sees.
439 | 
440 | In our setup, having a primary is the simplest option for this.
441 | ts so we don't see that it is impossible for there to be a supermajority for $B$ until late. 
442 | If $B$ is not the best block given the last finalised block but $B'$  with the same block number, they could stop eit
443 | 
444 | }
445 | 
446 | 
447 | 
448 | \section{ Analysis }
449 | 
450 | 
451 | To analyse the performance of our finality gadget, we will need versions of our properties that appropriately depend on time: 
452 | 
453 | \begin{itemize}
454 | \item{\bf Fast termination:} {\em If the last finalised block has number $n$ and, until another block is finalised, the best chain observed by all participants will include the same block with block number $n+1$, then a block with number $n+1$ will be finalised within time $T$.}
455 | \item{\bf Recent validity:} {\em If an honest voter finalises a block $B$ then that block was seen in the best chain observed by some honest voter containing some previously finalised ancestor of $B$ more recently than time $T$ ago.}
456 | \end{itemize}
457 | 
458 | Intuitively, fast termination implies that we finalise blocks fast as long as the block production mechanism achieves consensus fast whereas recent validity bounds the cost of starting to agree on something the block production mechanism's consensus later decides is not the best. In this case, we may waste time building on a chain that is never finalised so it is important to bound how long we do that.
459 | 
460 | These properties will typically only hold with high probability. In the asynchronous case, we would need to measure time in rounds of the protocol rather than seconds to make sense of these properties.  We are also interested in being able to remove and punish Byzantine voters, for which we will need:
461 | 
462 | \begin{itemize}
463 | 	\item{\bf Accountable Safety:} {\em If blocks on different chains are finalised, then we can identify at least $f+1$ Byzantine voters.}
464 | \end{itemize}
465 | 
466 | 
467 | 
468 | \subsection{ Accountable Safety}
469 | 
470 | The first thing we want to show is asynchronous safety, assuming we have at most $f$ Byzantine voters. This follows from the property that if $v$ sees round $r$ as completable then any block $B$ with $E_{r,v} \not\leq B$ has that it is impossible for one of $C_{r,v}$ or $V_{r,v}$ to have a supermajority for $B$ and so $B$ was not finalised in round $r$. This ensures that all honest prevotes and precommits in round $r+1$ are for chains that include any blocks that could have been finalised in round $r$. With an induction, this is what ensures that we cannot finalise blocks on different chains. To show accountable safety, we need to turn this proof around to show the contrapositive, when we finalise different blocks , then there are $f+1$ Byzantine voters. If we make this proof constructive, then it gives us a challenge procedure, that can assign blame to such voters.
471 | 
472 | \begin{theorem} \label{thm:accountable} If the protocol finalises any two blocks $B,B'$ for which valid commit messages were sent, but which do not lie on the same chain, then there are at least $f+1$ Byzantine voters who all voted in a particular vote. Furthermore, there is a synchronous procedure to find some such set $X$ of $f+1$ Byzantine voters.
473 | \end{theorem}
474 | 
475 | The challenge procedure works as follows: If $B$ and $B'$ are committed in the same round, then the union of their precommits must contain at least $f$ equivocations, so we are done.  Otherwise, we may assume by symmetry that $B$ was committed in round $r$ and $B'$ in round $r' > r$.  There are at least $n-f$ voters who precommitted $\geq B'$ or equivocated in round $r$ in their commit messages, so we ask those who precommitted $\geq B'$ why they did so.
476 | 
477 | Starting with $r''=r' $, we ask queries of the following form: 
478 | \begin{itemize}
479 | \item Why was $E_{r''-1} \not\geq B$  when you prevoted for or precommitted to $B'' \not\geq B$ in round $r'' > r$?
480 | \end{itemize}
481 | \noindent Any honest voter should be able to respond to this, as is shown in Lemma \ref{lem:honest-answer} below. 
482 | 
483 | The response is of the following form:
484 | \begin{itemize}
485 | \item A either a set $S$ of prevotes for round $r''-1$, or else a set $S$ of precommits for round $r''-1$, in either case such that it is impossible for $S$ to have a supermajority for $B$.
486 | \end{itemize}
487 | 
488 | Any honest voter should respond.  In particular, if no voter responds, then we consider all  voters how should have responded but didn't as Byzantine and we return this set of voters, along with any equivocators, which will be at least $n-f$ voters total. If any do respond, then if $r'' > r+1$, we can ask the same query for at least $n-f$ voters in round $r''-1$. We note however that if any voters do respond then we will not punish non-responders.
489 | 
490 | If we ask such queries for a vote in all rounds between $r''=r'$ and $r''=r+1$ and get valid responses, since some voter responds when $r''=r+1$, then we have either a set $S$ of prevotes or precommits in round $r$ that show it is impossible for $S$ to have a supermajority for $B$ in round $r$.
491 | 
492 | If $S$ is a set of precommits, then if we take the union of $S$ and the set of precommits in the commit message for $B$, then the resulting set of precommits for round $r$ has a supermajority for $B$ and it is impossible for it to have a supermajority for $B$. This is possible if the set is not safe and so there must be at least $f+1$ voters who equivocate an so are Byzantine.
493 | 
494 | If we get a set $S$ of prevotes for round $r$ that does not have a supermajority for $B$, then we need to ask a query of the form
495 | 
496 | \begin{itemize}
497 | \item Which prevotes for round $r$ have you seen?
498 | \end{itemize}
499 | \noindent to all the voters of precommit in the commit message for $B$  who voted for blocks $B'' \geq B$. There must be $n-f$ such voters and a valid response to this query is a set $T$ of prevotes for round $r$ with a supermajority for $B''$ and so a supermajority for $B$.
500 | 
501 | 
502 | If any give a valid response, by a similar argument to the above, $S \cup T$ will have $f+1$ equivocations.
503 | 
504 | So we either discover $f+1$ equivocations in a vote or else $n-f > f+1$ voters either equivocate or fail to validly respond like a honest voter could do to a query.
505 | 
506 | 
507 | \begin{lemma} \label{lem:honest-answer}
508 | An honest voter can answer the first type of query.
509 | \end{lemma}
510 | We first show that, if a prevote or precommit in round $r$ is cast by an honest voter $v$ for a block $B''$, then at the time of the vote we had $B'' \geq E_{r-1,v}$.
511 | Prevotes should be for the head of a chain containing either $E_{r-1,v}$ or some $B''' > E_{r-1,v}$ by step 2 or 3.  In either case we have $B'' \geq E_{r-1,v}$. Precommits should be for $g(V_{r,v})$ but $v$ waits until $g(V_{r,v}) \geq E_{r-1,v}$, by step 4, before precommitting, so again this holds.
512 | It follows that, if $B'' \not\geq B$, then we had $E_{r-1,v} \not\geq B$.
513 | 
514 | We next show that if we had $E_{r-1,v} \not\geq B$ at the time of the vote then we can respond to the query validly, by demonstrating the impossibility of a supermajority for $B$. 
515 | If $B$ was not on the same chain with $g(V_{r-1,v})$, then by Lemma \ref{lem:impossible} (iii), it was impossible for $V_{r-1,v}$ to have a supermajority for $B$, as desired. 
516 | If $B$ was on the same chain as $g(V_{r-1,v})$, then it was on the same chain as $E_{r-1,v}$ as well.  In this case, we must have $B > E_{r-1,v}$ since $E_{r-1,v} \not\geq B$.
517 | % ??????  
518 | % ... Need $E_{r-1,v} \geq g(V_{r-1,v})$ somehow?? ...
519 | % ??????
520 | % As $v$ started the round, we know $v$ sees $r-1$ as completable, so 
521 | %  either $E_{r-1,v} < g(V_{r-1,v})$ or it is impossible for $C_{r-1,v}$ to have a supermajority for any children of $g(V_{r-1,v})$, 
522 | % ??????
523 |  However, possibly using that round $r-1$ is completable, it was impossible for $C_{r-1,v}$ to have a supermajority for any child of $E_{r-1,v}$ on the same chain with $g(V_{v,r})$ and in particular for the child of $E_{r-1,v}$ on $\textrm{chain}(B)$.
524 | % ??????
525 | By Lemma \ref{lem:impossible} (i), this means $C_{r-1,v}$ did not have a supermajority for $B$, again as desired.
526 | 
527 | Thus we have that, at the time of the vote, for one of $V_{r-1,v}$, $C_{r-1,v}$, it was impossible to have a supermajority for $B$. The current sets $V_{r-1,v}$ and $C_{r-1,v}$ are supersets of those at the time of the vote, and so by Lemma \ref{lem:impossible} (ii), it is still impossible. Thus $v$ can respond validly.
528 | 
529 | 
530 | This is enough to show Theorem \ref{thm:accountable}.  Note that if $v$ sees a commit message for a block $B$ in round $r$ and has that $E_{r',v} \not\geq B$, for some completable round $r' \geq r$, then they should also be able to start a challenge procedure that successfully identifies at least $f+1$ Byzantine voters in some round. Thus we have that:
531 | 
532 | \begin{corollary} \label{cor:overestimate-final}
533 | If there at most $f$ Byzantine voters in any vote, $B$ was finalised in round $r$, and an honest participant $v$ sees that round $r' \geq r$ is completable, then $E_{r',v} \geq B$.
534 | \end{corollary}
535 | 
536 | \subsection{Liveness }
537 | 
538 | We show the protocol is deadlock free and also that it finalises new blocks quickly in a weakly synchronous model.
539 | For this section, we will assume that there are at most $f < n/3$ Byzantine voters for each vote, and so that the sets of prevotes and precommits for each round are safe.
540 | 
541 | We define $V_{r,v,t}$ be the set $V_{r,v}$ at time $t$ and similarly for $C_{r,v,t}$ and the block $E_{r,v,t}$ .
542 | 
543 | We first show that the completability of a round and the estimate for a completable round are monotone in the votes we see, in the latter case monotonically decreasing: 
544 | 
545 | \begin{lemma} \label{lem:message-monotonicity-completed-estimate}
546 | Let $v,v'$ be (possibly identical) honest participants, $t,t'$ be times, and $r$ be a round.
547 | Then if $V_{r,v,t} \subseteq V_{r,v',t'}$ and $C_{r,v,t} \subseteq C_{r,v',t'}$and $v$ sees that $r$ is completable at time $t$, then $E_{r,v',t'} \leq E_{r,v,t}$ and $v'$ sees that $r$ is completable at time $t'$.
548 | \end{lemma}
549 | 
550 | \begin{proof} 
551 | Since $v$ sees that $r$ is completable at time $t$, 
552 | either $E_{r,v} < g(V_{r,v})$ requiring $(n+f+1)/2 > 2f + 1$ votes, or else it is impossible for $C_{r,v}$ to have a supermajority for any children of $g(V_{r,v})$, requiring $2f + 1$ votes.
553 | In either case, both $V_{r,v,t}$ and $C_{r,v,t}$ contain votes from $2f + 1$ voters and so the same holds for $V_{r,v',t'}$ and $C_{r,v',t'}$. 
554 | By Lemma \ref{lem:ghost-monotonicity} (ii), $g(V_{r,v',t'}) \geq g(V_{r,v,t})$.
555 | As it is impossible for $C_{r,v,t}$ to have a supermajority for any children of $g(V_{r,v,t})$, it follows from Lemma \ref{lem:impossible} (i \& ii) that it is impossible for $C_{r,v',t'}$ as well, and so both $E_{r,v',t'} \leq g(V_{r,v,t})$ and $v'$ sees $r$ is completable at time $t'$.
556 | But now $E_{r,v,t}$ and $E_{r,v',t'}$ are the last blocks on $\textrm{chain}(g(V_{r,v,t}))$ for which it is possible for $C_{r,v,t}$ and $C_{r,v',t'}$ respectively to have a supermajority, 
557 | As it is possible for $C_{r,v',t'}$ to have a supermajority for $E_{r,v',t'}$, then it is possible for $C_{r,v,t}$ to have a supermajority for $E_{r,v',t'}$ as well, by Lemma \ref{lem:impossible} (ii) and tolerance assumptions, so $E_{r,v',t'} \leq E_{r,v,t}$.
558 | \end{proof}
559 |  
560 | \subsubsection{Deadlock Freeness}
561 | 
562 | Now we can show deadlock freeness for the asynchronous gossip network model, when a message that is sent or received by any honest participant is eventually received by all honest participants.
563 | 
564 | \begin{proposition} Suppose that we are in the asynchronous gossip network model and that at most $f$ voters for any vote are Byzantine. Then the protocol is deadlock free.\end{proposition}
565 | 
566 | \begin{proof} We need to show that if all honest participants reach some vote, then all of them eventually reach the next.
567 | 
568 | If all honest voters reach a vote, then they will vote and all honest participants see their votes. We need to deal with the two conditions that might block the algorithm even then.
569 | To reach the prevote of round $r$, a participant may be held up at the condition that round $r-1$ must be completable. To reach the precommit, a voter may be held up by the condition that $g(V_{r,v}) \geq E_{r-1,v}$.
570 | 
571 | For the first case, the prevote, let $S$ be the set of all prevotes from round $r-1$ that any honest voter saw before they precommitted in round $r-1$.
572 | By Lemma \ref{lem:ghost-monotonicity}, when voter $v'$ precommitted, they do it for block $g(V_{r-1,v'}) \leq g(S)$.
573 | Let $T$ be the set of precommits in round $r$ cast by honest voters.
574 | Then for any block $B \not\leq g(S)$, $T$ does not contain any votes that are $\geq B$ and so it is impossible for $T$ to have a supermajority for $B$.
575 | In particular, it is impossible for $T$ to have a supermajority for any child of $g(S)$. 
576 | 
577 | Now consider a voter $v$. By our network assumption, there is a time $t$ by which they have seen the votes in $S$ and $T$. Consider any $t' \geq t$.
578 | At this point we have $g(V_{r,v,t;}) \geq g(S)$. It is impossible for $C_{r,v,t'}$ to have a supermajority for any child of $g(S)$ and so $E_{r-1,v,t'} \leq g(S)$, whether or not this inequality is strict, we satisfy one of the two conditions for $v$ to see that round $r-1$ is completable at time $t'$.
579 | Thus if all honest voters reach the precommit vote of round $r-1$, all honest voters reach the prevote of round $r$.
580 | 
581 | Now we consider the second case, reaching the precommit. 
582 | Note that any honest prevoter in round $r$ votes for a block $B_v \geq E_{r-1,v,t_v}$ where $t_v$ is the time they vote. Now consider any honest voter for the precommit $v'$. By some time $t'$, they have received all the messages received by each honest voter $v$ at time $t_v$ and $v'$'s prevote. 
583 | Then by Corollary \ref{cor:overestimate-final}, $B_v \geq E_{r-1,v,t_v} \geq E_{r-1,v',t'}$. Since $V_{r,v',t'}$ contains these $B_v$, $g(V_{r,v',t'}) \geq  E_{r-1,v',t'}$. Thus if all honest voters prevote in round $r$, eventually all honest voters precommit in round $r$.
584 | 
585 | An easy induction completes the proof of the proposition.
586 | \end{proof}
587 | 
588 | \subsubsection{Weakly synchronous liveness}
589 | 
590 | Now we consider the weakly synchronous gossip network model. The idea that there is some global stabilisation time($\GST$) such that any message received or sent by an honest participant at time $t$ is received by all honest participants at time $\max\{t,\GST\}+T$.
591 | 
592 | Let $t_r$ be the first time any honest participant enters round $r$ i.e. the minimum over honest participants $v$ of $t_{r,v}$.
593 | 
594 | \begin{lemma} \label{lem:timings}
595 | Assume the weakly synchronous gossip network model and that each vote has at most $f$ Byzantine voters. Then if $t_r \geq \GST$, we have that
596 | \begin{itemize}
597 | \item[(i)] $t_r \leq t_{r,v} \leq t_r+T$ for any honest participant $v$,
598 | \item[(ii)] no honest voter prevotes before time $t_r+2T$,
599 | \item[(iii)] any honest voter $v$ precommits at the latest at time $t_{r,v}+4T$,
600 | \item[(iv)] for any honest $v$, $t_{r+1,v} \leq t_r + 6T$.
601 | \end{itemize}
602 | \end{lemma}
603 | 
604 | 
605 | \begin{proof} Let $v'$ be one of the first honest participants to enter round $r$ i.e. with $t_{r,v'}=t_r$. 
606 | By our network assumption, all messages received by $v'$ before they ended are received by all honest participants before time $t_r+T$.
607 | In particular at time $t_r$, $v'$ sees that all previous rounds are completable and so by Corollary \ref{cor:overestimate-final}, so does every other honest participant by time $t_r+T$.
608 | Also since for $r' < r$, at some time $s_{r'} \leq t_r$ $g(V_{r',v',s_r'}) \geq E_{r',v',s_r'}$, again by Lemma 4, for all honest $v$, $g(V_{r',v,t_r+T}) \geq E_{r',v,t_r+T}$. Looking at the conditions for voting, this means that any honest voter does not need to wait before voting in any round $r' \leq r$. 
609 | Thus they cast any remaining votes and enter round $r$ by time $t_r + T$. This shows (i).
610 | 
611 | For (ii), note that the only reason why an honest voter would not wait until time $t_{r,v}+2T \geq t_r+ 2T$ is when $n-f$ voters have already prevoted. But since some of those $n-f$ votes are honest, this is impossible before $t_r+2T$
612 | 
613 | Now an honest voter $v''$ prevotes at time $t_{r,v''}+2T \leq t_r +3T$ and by our network assumptions all honest participants receive this vote by time $t_r+4T$. An honest voter for the precommit $v$ has also received all messages that $v''$ received before they prevoted by then.
614 | Thus the block they prevoted has $B_{v''} \geq E_{r-1,v''} \geq E_{r-1,v,t_r+4T}$, since this holds for every honest voter $v''$, $g(V_{r,v,t_r+4T}) \geq E_{r-1,v,t_r+4T}$. Thus they will precommit by time $t_{r,v}+4T$ which shows (iii).
615 | 
616 | By the network assumption an honest voter $v'$'s precommit will be received by all honest participants $v$ by time $t_{r,v'}+ 5T \leq t_r+6T$.
617 | Since $v$ will also have received all prevotes $v$ say when they precommitted by this time, their vote $B_{v'}$ will have $B_{v'}=g(V_{r,v'}) \leq g(V_{r,v,t_r+6T})$.
618 | Thus $C_{r, v, t_r+6T}$ contains precommits from $n-f$ voters $v'$ with $B_{v'} \leq g(V_{r,v,t_r+6T})$ and thus it is impossible for $C_{r,v,t_r+6T}$ to have a supermajority for any children of $g(V_{r,v, t_r+6T})$.
619 | Thus $v$ sees that round $r$ is completable at time $t_r+6T$. Since they have already prevoted and precommitted if they were a voter, they will move to round $r+1$ by at latest $t_t+6T$. This is (iv).
620 | \end{proof}
621 | 
622 | \begin{lemma} \label{lem:honest-prevote-timings}
623 | Suppose $t_r \geq \GST$ and very vote has at most $f$ Byzantine voters. Let $H_r$ be the set of prevotes ever cast by honest voters in round $r$. Then
624 | \begin{itemize}
625 | \item[(a)] any honest voter precommits to a block $\geq  g(H_r)$,
626 | 
627 | \item[(b)] every honest participant finalises $g(H_r)$ by time $t_r+6T$.
628 | \end{itemize}
629 | \end{lemma}
630 | 
631 | \begin{proof} For (a), we separate into cases based on which of the conditions (i)-(iii) that we wait for to precommit hold.
632 | 
633 | For (i), all honest voters prevote in round $r$ by time $t_r+3T$. So any honest voter $v$ who precommits at or after time $t_{r,v}+4T \geq t_r+4T$ has received all votes in $H_r$ and by Lemma \ref{lem:ghost-monotonicity}, precommits to a block $\geq g(H_r)$.
634 | 
635 | For (ii), we argue that no honest voter commits a block $\not\geq g(H_r)$ first. The result will then follow by an easy induction once the other cases are dealt with. Suppose that no honest voter has precommitted a block $\not \geq g(H_r)$ so far and that a voter $v$ votes early because of (ii).
636 | 
637 | Note that, since we assume that all precommits by honest voters so far were $\geq g(H_r)$, it is possible for $C_{r,v}$ to have a supermajority for $g(H_r)$.
638 | For (ii) to hold for a voter $v$ i.e for round $r$ to be completable, it must be the case that either it is impossible for $C_{r,v}$ to have a supermajority for $g(V_{r,v})$ or else be impossible for $C_{r,v}$ to have a supermajority for any children of $g(V_{r,v})$. By Lemma \ref{lem:impossible} cannot have $g(V_{r,v}) < g(H_r)$.
639 | But by Lemma \ref{lem:ghost-monotonicity}, these are on the same chain and so $g(V_{r,v}) \geq g(H_r)$. Since this is the block $v$ precommits to, we are done in case (ii)
640 | 
641 | For (iii), let $v$ be the voter in question. Note that since $n-f$ honest voters prevoted $\geq g(H_r)$, it is possible for $V_{r,v}$ to have a supermajority for $g(H_r)$. By Lemma \ref{lem:ghost-monotonicity}, $g(V_{r,v})$ is on the same chain as $g(H_r)$.
642 | For (iii), it is impossible for $V_{r,v}$ to have a supermajority for any children of $g(V_{r,v})$. If we had $g(V_{r,v}) < g(H_r)$, by Lemma \ref{lem:impossible}, this would mean that it would be impossible for $V_{r,v}$ to have a supermajority for $g(H_r)$ as well. So it must be that $g(V_{r,v} )\geq g(H_r)$ as required.
643 | 
644 | For (b), combining (a) and Lemma \ref{lem:timings} (iii), we have that any honest voter $v$ precommits $\geq g(H_r)$ by time $t_{r,v}+4T$. By our network assumption, all honest participants receive these precommits by time $t_r+6T$ and so finalise $g(H_r)$ if they have not done so already.
645 | \end{proof}
646 | 
647 | \begin{lemma} \label{lem:primary-finalises}
648 |  Suppose that $t_r \geq \GST$, the primary $v$ of round $r$ is honest and no vote has more than $f$ Byzantine voters. Let $B=E_{r-1,v,t_{v,r}}$ be the block $v$ broadcasts if it is not final. Then every honest prevoter prevotes for the best chain including $B$ and all honest voter finalise $B$ by time $t_r+6T$.
649 |  \end{lemma}
650 | 
651 | \begin{proof} By Lemma \ref{lem:timings} and our network assumptions, no honest voter  prevotes before time $t_r+2T \geq t_{r,v}+2T$ and so at this time, they will have seen all prevotes and precommits seen by $v$ at $t_{r,v}$ and the block $B$ if $v$ broadcast it then. By Lemma \ref{lem:message-monotonicity-completed-estimate}, any honest voter $v'$ has $E_{r-1,v'} \leq B \leq g(V_{r-1,v})$ then.
652 | 
653 | So if the primary broadcast $B$, then $v'$ prevotes for the best chain including $B$. If the primary did not broadcast $B$, then they finalise it. By Corollary \ref{cor:overestimate-final}, it must be that $E_{r-1,v'} \geq B$ and so $E_{r-1,v'}=B$ and so in this case $v'$ also prevotes for the best chain including $B$.
654 | 
655 | Since all honest voters prevote $\geq B$, $g(H_r) \geq B$ and so by Lemma \ref{lem:honest-prevote-timings}, all honest participants finalise $B$ by time $t_r+6T$
656 | \end{proof}
657 | 
658 | 
659 | 
660 | \begin{lemma}
661 |  Suppose that $t_r \geq \GST+T$ and the primary of round $r$ is honest. 
662 | Let $B$ be the latest block that is ever finalised in rounds  $<r$ (even if no honest participant finalises it until after $t_r$). If all honest voters for the prevote in round $r$ agree that the best chain containing $B$ include the same child $B'$ of $B$, then they all finalises some child of $B$ before $t_r+6T$.
663 | \end{lemma}
664 | 
665 | \begin{proof} By Corollary \ref{cor:overestimate-final}, any honest participant sees that $E_{r-1} \geq B$ during round $r$. Let $v$ be the primary of round $r$ and $B''=E_{r-1,v,t_{r,v}}$. If $B'' > B$, then by Lemma \ref{lem:primary-finalises}, all honest participants finalise $B''$ by time $t_r+6T$ which means they finalised a child of $B$. If $B''=B$, then by Lemma \ref{lem:honest-prevote-timings}, all honest voters prevote for the best chain including $B$.
666 | By assumption these chains include $B'$ and so $g(H_r) \geq B$. By Lemma \ref{lem:honest-prevote-timings}, this means that $B'$ is finalised by time $t_r+6T$.
667 | \end{proof}
668 | 
669 | 
670 | 
671 | 
672 | 
673 | 
674 | 
675 | \subsubsection{Recent Validity}
676 | 
677 | \begin{lemma} \label{lem:honest-recent-validity}
678 | Suppose that $t_r \geq \GST$, the primary of round $r$ is honest and all votes have at most $f$ Byzantine voters.
679 | Let $B$ be a block that less than $f+1$ honest prevoters in round $r$ saw as being in the best chain of an ancestor of $B$ at the time they prevoted.
680 | Then either all honest participants finalise $B$ before time $t_r+6T$ or no honest participant ever has $g(V_{r,v}) \geq B$ or $E_{r,v} \geq B$.
681 | \end{lemma}
682 | 
683 | \begin{proof} Let $v'$ be the primary of round $r$ and let $B'=E_{r-1,v',t_{r,v'}}$. If $B' \geq B$, then by Lemma \ref{lem:primary-finalises}, all honest participants finalise $B$ by time $t_r+6T$. If $B' \not\geq B$, then by Lemma \ref{lem:primary-finalises}, at most $f$ honest voters prevotes $\geq B$. In this case, less than $2f+1 \leq (n+f+1)/2$ prevoters vote $\geq B$ or equivocate and so no honest participant ever has $g(V_{r,v}) \geq B$.
684 | \end{proof}
685 | 
686 | 
687 | \begin{corollary} For $t - 6T > t' \geq \GST$, suppose that an honest participant finalises $B$ at time $t$ but that no honest voter has seen $B$ as in the best chain containing some ancestor of $B$ in between times $t'$ and $t$, then at least $(t-t')/6T - 1$ rounds in a row had Byzantine primaries. \end{corollary}
688 | 
689 | 
690 | 
691 | \bibliography{grandpa} % net,os,sec,soc,theory, 
692 | 
693 | \end{document}
694 | 


--------------------------------------------------------------------------------