14 |
15 |
16 |
17 |
18 |
19 |
--------------------------------------------------------------------------------
/_layouts/home.html:
--------------------------------------------------------------------------------
1 | ---
2 | layout: default
3 | ---
4 |
5 | {{ content }}
6 |
--------------------------------------------------------------------------------
/_layouts/page.html:
--------------------------------------------------------------------------------
1 | ---
2 | layout: default
3 | ---
4 |
5 |
6 |
10 |
--------------------------------------------------------------------------------
/_layouts/post.html:
--------------------------------------------------------------------------------
1 | ---
2 | layout: default
3 | ---
4 |
5 |
6 |
15 |
--------------------------------------------------------------------------------
/_layouts/pr.html:
--------------------------------------------------------------------------------
1 | ---
2 | layout: default
3 | ---
4 |
5 | {% capture components %}
6 | {%- for comp in page.components -%}
7 | {{comp}}{% unless forloop.last %}, {% endunless %}
8 | {%- endfor -%}
9 | {% endcapture %}
10 |
11 | {% capture authors %}
12 | {%- for author in page.authors -%}
13 | {{author}}
14 |
15 | {% unless forloop.last %}, {% endunless %}
16 | {%- endfor -%}
17 | {% endcapture %}
18 |
19 |
20 |
70 |
--------------------------------------------------------------------------------
/_plugins/auto_logs_markup.rb:
--------------------------------------------------------------------------------
1 | # frozen_string_literal: true
2 |
3 | require 'uri'
4 |
5 | module Jekyll
6 | class IRCBlock < Liquid::Block
7 | # Convert IRC meeting logs from plain text to HTML with functional links
8 | # and log lines by enclosing them in {% irc %} and {% endirc %} tags.
9 |
10 | # URI schemes to accept for extraction.
11 | URI_SCHEMES = %w(http https).freeze
12 |
13 | # Trailing chars (all punctuation except "/") to remove from URIs.
14 | TRAILING = /[^\/[:^punct:]]+$/
15 |
16 | # Regex to select lines starting with "HH:MM " time.
17 | HH_MM = /^([0-1][0-9]|[2][0-3]):[0-5][0-9] .*/
18 |
19 | # Regex to select IRC .
20 | IRC_NICK = /^(?:\s*)(<.+?>)/
21 |
22 | # Regex to select "<" and ">" chars.
23 | LT_GT = /[<>]/
24 |
25 | # Maximum digits to display for log lines.
26 | LINE_DIGITS = 3
27 |
28 | # Length of timestamp string being used.
29 | TIME_SIZE = 'HH:MM'.size
30 | TIME_SIZE_PLUS_1 = TIME_SIZE + 1 # Micro-perf optimization
31 |
32 | NON_BREAKING_SPACE = ' '
33 |
34 | COLORS = %w(brown goldenrod cadetblue chocolate cornflowerblue coral crimson
35 | forestgreen darkblue firebrick blue green grey hotpink indianred indigo
36 | blueviolet maroon mediumblue mediumpurple mediumseagreen navy fuchsia
37 | olive orchid purple red seagreen sienna orange slateblue peru salmon teal
38 | magenta steelblue rebeccapurple tomato violet darkcyan).freeze
39 |
40 | NUM_COLORS = COLORS.size.freeze
41 |
42 | def initialize(tag_name, text, tokens)
43 | super
44 | end
45 |
46 | def render(context)
47 | output = super
48 |
49 | # Reset color data for each post. Seed the index with part of the content.
50 | colors, color_index = {}, (output[-256..-250] || '').bytes.sum
51 |
52 | # Loop through each line of the meeting logs.
53 | output.gsub!(HH_MM).with_index(1) do |line, index|
54 |
55 | # Separate the log line into useful parts.
56 | lineno = "#{NON_BREAKING_SPACE * (LINE_DIGITS - index.to_s.size)}#{index}"
57 | time = line[0..TIME_SIZE]
58 | name = IRC_NICK.match(line[TIME_SIZE_PLUS_1..-1]).to_s
59 | nick = name.gsub(LT_GT, '').strip
60 | color = colors[nick] || (color_index = (color_index + 1) % NUM_COLORS ;
61 | colors[nick] = COLORS[color_index])
62 | nick = "<#{nick}>" unless nick == ''
63 | message = CGI.escapeHTML(line[TIME_SIZE_PLUS_1 + name.size..-1])
64 |
65 | # Extract URIs from the message and convert them to HTML links.
66 | URI.extract(message, schemes = URI_SCHEMES).each do |uri|
67 | link = uri.sub(TRAILING, '') # Strip unwanted trailing punctuation
68 | message.sub!(link, "#{link}")
69 | end
70 |
71 | # Return the log line as HTML markup.
72 | "" \
73 | "
"
82 | end
83 |
84 | output
85 | end
86 | end
87 | end
88 |
89 | Liquid::Template.register_tag('irc', Jekyll::IRCBlock)
90 |
--------------------------------------------------------------------------------
/_posts/2019-05-22-#15450.md:
--------------------------------------------------------------------------------
1 | ---
2 | layout: pr
3 | title: "Create wallet menu option"
4 | components: ["gui"]
5 | pr: 15450
6 | authors: [achow101]
7 | host: jnewbery
8 | status: past
9 | ---
10 |
11 | ## Notes
12 |
13 | - Bitcoin uses QT for its GUI. The minimum version is 5.5. [Online documentation](https://doc.qt.io/archives/qt-5.5/index.html).
14 | - This PR is one of the last in a series of multiwallet PRs. The first of those was [#8694](https://github.com/bitcoin/bitcoin/pull/8694) which added multiwallet support through the RPC interface and was merged in June 2017.
15 | - Adding the various multiwallet functions (create/open/close wallet) at runtime is tracked by [Issue 13059](https://github.com/bitcoin/bitcoin/issues/13059).
16 | - We already have UI support for opening and closing wallets. This PR adds UI support for creating wallets.
17 | - Creating wallets dynamically is relatively new functionality (merged in [#13058](https://github.com/bitcoin/bitcoin/pull/13058) in June 2018). There have been a couple of additional options added to the `createwallet` RPC since it was added: [Disable private keys](https://github.com/bitcoin/bitcoin/pull/9662), [Blank wallets](https://github.com/bitcoin/bitcoin/pull/15226) and [born-encrypted wallets](https://github.com/bitcoin/bitcoin/pull/15006).
18 | - This PR was opened before the born-encrypted wallets PR was merged, so potentially could be simplified now that the functionality exists within the wallet component.
19 | - For GUI PRs, I always like to build and manually test the PR to get a sense of what the new UI functionality feels like. If you're testing on the same machine that you run your mainnet node on, REMEMBER TO MOVE OR BACKUP YOUR WALLET/DATADIR BEFORE TESTING!
20 | - Feel free to leave feedback on UI PRs about usability as well as comments on the code.
21 |
22 | ## Meeting Log
23 |
24 | {% irc %}
25 | 13:02 < jnewbery> hi!
26 | 13:02 < dmkathayat_> Hi!
27 | 13:02 < davterra> Hi!
28 | 13:02 < ariard> hi
29 | 13:03 < peevsie> yo
30 | 13:03 < jnewbery> Today's PR is https://github.com/bitcoin/bitcoin/pull/15450
31 | 13:03 < pinheadmz> buenos dias
32 | 13:03 < emzy> Hi
33 | 13:03 < michaelfolkson> What up. Head spinning trying to work out how all these PRs overlap lol
34 | 13:03 < jnewbery> I wrote my review notes here: https://bitcoin-core-review-club.github.io/15450.html to save us some time at the start of the meeting
35 | 13:03 < jonatack> hi
36 | 13:04 < jnewbery> So rather than me talking for 5 minutes now, let's just get straight into questions
37 | 13:04 < jonatack> is no testing for qt a thing
38 | 13:05 < jnewbery> Most qt PRs don't include tests
39 | 13:05 < jnewbery> But there are some in src/qt/test
40 | 13:06 < MarcoFalke> Yeah, and those are more like gui unit tests, not actual gui tests
41 | 13:07 < MarcoFalke> Currently the only way to test the GUI is by running it by hand
42 | 13:08 < michaelfolkson> So to confirm I understand what is going on. You can currently create a new independent HD wallet using bitcoind but until now you couldn't using Qt. Qt can't leverage bitcoind RPC so the C++ code needs to be written afresh for Qt in addition to editing the GUI?
43 | 13:10 < jnewbery> michaelfolkson: we try to make the RPC layer as thin as possible so functionality can be shared between RPC and QT. Here's the createwallet RPC: https://github.com/bitcoin/bitcoin/blob/1c177c3a004f91eca743bb3a0dd9534a544026d5/src/wallet/rpcwallet.cpp#L2642
44 | 13:10 < jnewbery> You can see that it's mostly just parsing arguments and then calling CWallet::CreateWalletFromFile() which does the heavy lifting
45 | 13:11 < jnewbery> This PR adds a new method to the node interface called createWallet() which basically does the same stuff
46 | 13:11 < jnewbery> https://github.com/bitcoin/bitcoin/pull/15450/files#diff-7e7292143c1cc2b4125d85f1ae5cf046R261
47 | 13:12 < jonatack> thanks! i suppose then that units are added for any new non-qt code. seem might still be good to add e2e tests for the user flows.
48 | 13:12 < jonatack> unit tests
49 | 13:12 < jnewbery> jonatack: Yes, I think that would be valuable contribution
50 | 13:12 < jonatack> end to end functional ones?
51 | 13:12 < jnewbery> yes
52 | 13:12 < jonatack> ty
53 | 13:13 < jnewbery> I don't know anything about testing QT
54 | 13:13 < jnewbery> But I imagine there's documentation on the QT website
55 | 13:13 < MarcoFalke> Indeed, I'd welcome any progress in that direction
56 | 13:13 < jnewbery> michaelfolkson: to follow up on my comment in the review notes "This PR was opened before the born-encrypted wallets PR was merged, so potentially could be simplified now that the functionality exists within the wallet component.
57 | 13:13 < MarcoFalke> I don't know if there are existing framworks
58 | 13:14 < jnewbery> You can see here: https://github.com/bitcoin/bitcoin/pull/15450/files#diff-fef859e81d3321e85221891fd768efadR75 that there's some duplication of logic for creating an encrypted wallet
59 | 13:16 < jnewbery> that logic now exists in the wallet RPC: https://github.com/bitcoin/bitcoin/blob/master/src/wallet/rpcwallet.cpp#L2710 . Perhaps this could be factored out of there into wallet.cpp and then shared by the RPC and QT
60 | 13:16 < michaelfolkson> : Thanks, that's useful. I wasn't expecting to see any C++ code for this PR, I was expecting to just be looking at GUI design. But I forgot that Qt wasn't just a front end
61 | 13:16 < jnewbery> I'm not entirely sure of that though. I haven't checked whether there are subtle differences in what logic is required for QT and RPC
62 | 13:17 < jnewbery> Yep, it's all C++
63 | 13:19 < jnewbery> Did everyone manage to build the PR and do some manual testing?
64 | 13:19 < jnewbery> Any questions about that?
65 | 13:20 < jnewbery> I guess that's it for today. Short meeting!
66 | 13:20 < jnewbery> Before we go, does anyone have requests for future PRs to cover?
67 | 13:21 < jonatack> i compiled but didn't run it yet because i didn't see it in root or source
68 | 13:21 < jonatack> any pointers ?
69 | 13:21 < jnewbery> jonatack: it should be at src/qt/bitcoin-qt
70 | 13:21 < jnewbery> if not, perhaps you're not configured to build the qt binary. Check the output from when you ran `configure`
71 | 13:22 < jnewbery> You should see: 'with gui / qt = yes'
72 | 13:22 < jonatack> all good
73 | 13:22 < emzy> I had no time to do it. Maybe I will check it later.
74 | 13:22 < michaelfolkson> For the creating blank empty wallets PR, what's the motivation for doing this? Importing a seed from elsewhere?
75 | 13:22 < jonatack> i never run qt, thanks for the tpi
76 | 13:22 < jonatack> tip
77 | 13:22 < michaelfolkson> #15226
78 | 13:23 < jnewbery> michaelfolkson: by default, a new wallet will generate an HD seed
79 | 13:23 < MarcoFalke> To follow up on the "How to test the GUI" question earlier: The only framework I am aware of is used by operating systems: OpenQA. I think it is a Open Suse in-house product. See https://openqa.opensuse.org/tests/940238#step/bootloader/3
80 | 13:23 < jnewbery> if you want to import your own seed, you can create a blank wallet and then import
81 | 13:24 < jnewbery> another motivation: when you encrypt the wallet, it'll create a new seed, so if you created a wallet that wasn't blank, you'd have a seed from before encryption and then a seed from after encryption, which perhaps you don't want
82 | 13:25 < jnewbery> #15226 also made #15006 (Add option to create an encrypted wallet) much simpler
83 | 13:25 < jonatack> thanks marco
84 | 13:25 < ariard> I had a a more general question which is on travis build process, I often get a timeout error "Error! Initial build successful, but not enough time..." What's the process to do after that ? Force-push new branch tip?
85 | 13:26 < MarcoFalke> Created an issue for that https://github.com/bitcoin/bitcoin/issues/16075
86 | 13:26 < MarcoFalke> jonatack: et al ^
87 | 13:26 < ariard> Do this also mean things have been modified which make tests longer to run ?
88 | 13:26 < jonatack> MarcoFalke: nice
89 | 13:26 < jnewbery> ariard: Yes, I believe force-pushing the branch will trigger a rebuild on travis
90 | 13:27 < jnewbery> just run `git commit --amend` to update the timestamp and then `git force push`
91 | 13:27 < jonatack> today's pr has seen no activity since 3 months.
92 | 13:28 < ariard> jnewbery: thanks for tip!
93 | 13:28 < jonatack> does achow have too much on his plate, does he need more help
94 | 13:28 < jonatack> or is it not a priority
95 | 13:29 < jnewbery> It might just not be a priority for him. I know he has a PR for a descriptor-based wallet, which is large and more important
96 | 13:29 < jonatack> right
97 | 13:30 < jnewbery> In general, if you see PRs that haven't had attention from their author for a while, there's nothing wrong in offering to take over maintaining the PR
98 | 13:32 < jnewbery> any other questions?
99 | 13:32 < jonatack> MarcoFalke:
100 | 13:32 < jonatack> do you plan to resubmit a pr
101 | 13:32 < jonatack> for the f tests speedup
102 | 13:32 < jonatack> you had 2
103 | 13:32 < michaelfolkson> achow is overseeing the Bitcoin Core hardware wallet interface too isn't he? That is ongoing
104 | 13:33 < MarcoFalke> jonatack: Good question. Not sure if we are allowed to discuss other prs here
105 | 13:33 < MarcoFalke> jnewbery: ? ^
106 | 13:33 < jonatack> related to ariard's q
107 | 13:33 < jnewbery> Is it directly related to this week's PR? If not, I think #bitcoin-core-dev is the better place to discuss it (so all interested parties can see it)
108 | 13:34 < jonatack> ok
109 | 13:35 < jonatack> qt is now running for me ty for the help jnewbery
110 | 13:36 < jnewbery> michaelfolkson: yes, achow maintains HWI
111 | 13:36 < jnewbery> jonatack: great!
112 | 13:36 < jnewbery> ok, unless there are any final questions, let's wrap it up.
113 | 13:37 < jnewbery> Thanks all! See you next week
114 | 13:37 < emzy> Tnx!
115 | 13:37 < jonatack> cheers! thanks everyone
116 | 13:38 < michaelfolkson> A quick final question? :)
117 | 13:38 < jnewbery> go for it!
118 | 13:39 < jnewbery> michaelfolkson: was 'A quick final question?' the final question or is there a question after that question?
119 | 13:40 < michaelfolkson> So what are next steps for advancing this? Someone who understands the code for the GUI needs to oversee all the merge conflicts between these different PRs?
120 | 13:40 < michaelfolkson> Sorry yes
121 | 13:40 < jnewbery> Which different PRs? This one doesn't currently conflict with anything
122 | 13:40 < michaelfolkson> And there needs to be more feedback on the design. I saw Sjors gave some suggestions for what the UI should be
123 | 13:41 < michaelfolkson> I was seeing a bunch of conflicts on one of the PRs you linked to
124 | 13:42 < jnewbery> Yeah, i think it's still appropriate to give design feedback at this stage
125 | 13:42 < jonatack> michael: perhaps adding a recent review can kick the pr back into action
126 | 13:42 < jnewbery> I agree. achow will probably be more motivated to work on it if he knows there are people keen to review
127 | 13:44 < michaelfolkson> Ok cool. It is just design considerations now right?
128 | 13:44 < jnewbery> No, the code needs review too
129 | 13:44 < jnewbery> all aspects of the PR
130 | 13:45 < michaelfolkson> Ok understood, thanks
131 | 13:45 < michaelfolkson> Ok I'll let you go, many thanks
132 | 13:46 < jnewbery> ok, thanks again. See you all next time!
133 | {% endirc %}
134 |
--------------------------------------------------------------------------------
/_posts/2019-09-11-#16688.md:
--------------------------------------------------------------------------------
1 | ---
2 | layout: pr
3 | title: "Add validation interface logging"
4 | components: ["utils/log/libs"]
5 | pr: 16688
6 | authors: [jkczyz]
7 | host: jkczyz
8 | status: past
9 | ---
10 |
11 | ## Notes
12 |
13 | - The
14 | [validation layer](https://github.com/bitcoin/bitcoin/blob/0d20c42a014ff95aab1447a92605c3a194cfeecc/src/validation.cpp)
15 | enforces Bitcoin's consensus rules.
16 | - It processes blocks and transactions from the net processing layer and updates the validated
17 | chain state (`CChainState`) and mempool (`CTxMemPool`).
18 | - Clients can subscribe to updates from the validation layer using the [`CValidationInterface`](https://github.com/bitcoin/bitcoin/blob/0d20c42a014ff95aab1447a92605c3a194cfeecc/src/validationinterface.h#L56).
19 | - The validation layer notifies `CValidationInterface` subscribers whenever the mempool or chainstate is updated. Some interface methods are (see *validationinterface.cpp* for the full list of interface methods):
20 | - [`UpdatedBlockTip`](https://github.com/bitcoin/bitcoin/blob/0d20c42a014ff95aab1447a92605c3a194cfeecc/src/validationinterface.h#L87) (fired once when the block chain is extended with one or more new blocks)
21 | - [`BlockConnected`](https://github.com/bitcoin/bitcoin/blob/0d20c42a014ff95aab1447a92605c3a194cfeecc/src/validationinterface.h#L111) (fired once for every block added to the block chain. The notification includes the serialized block with transactions)
22 | - [`TransactionAddedToMempool`](https://github.com/bitcoin/bitcoin/blob/0d20c42a014ff95aab1447a92605c3a194cfeecc/src/validationinterface.h#L93) (fired whenever a new transaction is accepted into the mempool. The notification includes the serialized transaction)
23 |
24 | ## Questions
25 |
26 | - What's the reason for adding logging to event notification?
27 | - What mechanism is used to notify listeners of validation events?
28 | - What code implements `CValidationInterface`? What does it use event
29 | notifications for?
30 | - What is a `CBlockLocator`? How is it used in this PR? How else is it used?
31 | - What refactorings are included in this PR?
32 | - How can this PR be tested? Could any other tests have been added?
33 |
34 | ## Meeting Log
35 |
36 | {% irc %}
37 | 13:00 < jkczyz> hi
38 | 13:00 < sosthene> nah it certainly comes from my side, happens all the time
39 | 13:00 < lightlike> hi
40 | 13:00 < jnewbery> hi
41 | 13:00 < b10c> hi
42 | 13:00 < udiWertheimer> jnewbery: that’s dedication!
43 | 13:00 < emzy> Hi
44 | 13:01 < jkczyz> As jnewbery noted, I'm guest hosting today. He may be around to help with the finer points :)
45 | 13:01 < jnewbery> udiWertheimer: haha. I'm going to duck out in a few minutes. Just wanted to make sure jkczyz has everything he needs to get started
46 | 13:01 < jkczyz> Today we’re reviewing PR #16688, which adds logging to validation layer’s event notification
47 | 13:01 < jkczyz> The validation layer processes blocks and transactions from the net processing layer and updates the validated chain state and mempool
48 | 13:02 < jkczyz> Any initial thoughts on this PR?
49 | 13:03 < jnewbery> I think it's great, and definitely going to be helpful for debugging issues as we move towards a more modular architecture
50 | 13:04 < jkczyz> jnewbery: Glad I can help in even a small way! While this is a fairly simple change, there are some lessons we can learn about the code through it
51 | 13:05 < jkczyz> Alright, let's jump into the questions
52 | 13:05 < jkczyz> What’s the reason for adding logging to event notification?
53 | 13:06 < sosthene> I guess it helps debugging?
54 | 13:07 < jkczyz> sosthene: correct, what sort of issues can it help debug?
55 | 13:07 < jnewbery> specifically, since the CValidationInterface is asynchronous, it can help with debugging threading issues and race conditions
56 | 13:07 < lightlike> it helps with error analysis, especially in case of race conditions, which appear in functional tests sometimes.
57 | 13:07 < emzy> There was a race condition where it would had helpd to debug.
58 | 13:07 < sosthene> In fact I wonder why there wasn't any log to begin with, as it seems a pretty critical part of Bitcoin to me.
59 | 13:08 < jkczyz> jnewbery, emzy: correct! Specifically it could have helped with issue #12978
60 | 13:08 < jkczyz> https://github.com/bitcoin/bitcoin/issues/12978
61 | 13:08 < lightlike> ironically, there is a is a race condition in the TSAN Travis build of this PR (which seems unrelated though)
62 | 13:08 < jkczyz> Here there was a race condition resulting in UpdatedBlockTip() signals delivered out of order
63 | 13:08 < jnewbery> sosthene: I agree this is critical, but for historic context, the CValidationInterface was updated about a year ago by Matt. Before then, I think it wasn't asynchronous (I may be wrong)
64 | 13:09 < jnewbery> bitcoind is (mostly) single threaded on the Message Handler thread. As we try to make it more multi-threaded, better logging at the interfaces is going to be more important
65 | 13:10 < jkczyz> lightlike: Yes, appears to be so. I thought about force pushing to see if there was some flakiness to the test
66 | 13:10 < jnewbery> I've got to head out now. Bye!
67 | 13:11 < jkczyz> jnewbery: Bye! Thanks for stopping in
68 | 13:11 < jkczyz> ok, next question: What mechanism is used to notify listeners of validation events?
69 | 13:11 < lightlike> jkczyz: i looked at it a little. seems related to https://github.com/bitcoin/bitcoin/issues/16133
70 | 13:12 < jkczyz> lightlike: cool, thanks for investigating! I'll have to look at that issue
71 | 13:13 < jkczyz> Hint: it uses a third-party library
72 | 13:15 < lightlike> don't know about it in depth, but via callback functions
73 | 13:16 < sosthene> I'm not even sure to quite get the question
74 | 13:16 < jkczyz> lightlike: yes, it uses callbacks, specifically with the boost:signals2 library
75 | 13:16 < jkczyz> https://www.boost.org/doc/libs/1_71_0/doc/html/signals2.html
76 | 13:17 < jkczyz> sosthene: The code in question is a bit tricky to reason through. I had to draw out a diagram to be honest!
77 | 13:17 < jkczyz> https://github.com/bitcoin/bitcoin/blob/d0f81a96d9c158a9226dc946bdd61d48c4d42959/src/validationinterface.cpp
78 | 13:17 < jkczyz> Ok, let's move on to some more interesting questions
79 | 13:18 < jkczyz> What code implements CValidationInterface? What does it use event notifications for?
80 | 13:18 < jkczyz> There are many correct answers to this one
81 | 13:19 < lightlike> the wallet does, for tracking the status of wallet transactions
82 | 13:21 < jkczyz> lightlike: yes, though this interface is adapted by NotificationsHandlerImpl for use by the wallet
83 | 13:21 < jkczyz> https://github.com/bitcoin/bitcoin/blob/9ab9d6356938ffe069ffc005a371d5405976823e/src/interfaces/chain.cpp#L158
84 | 13:22 < sosthene> what about the networking part? I guess you need to know if a peer is feeding you invalid blocks or tx?
85 | 13:24 < jkczyz> sosthene: correct, PeerLogicValidation (net processing layer) makes use of notificaitons to send peers new data and to validate peer messages (I believe :)
86 | 13:25 < jkczyz> On the wallet-side, using event notification helps with separation betweeen node and wallet functionality
87 | 13:25 < jkczyz> anything else?
88 | 13:25 < sosthene> I see it implemented in the BaseIndex class too, but I'm not too sure about what it does
89 | 13:25 < sosthene> https://github.com/bitcoin/bitcoin/blob/e4beef611a423e9f836fa210a51634e94f14d830/src/index/base.h
90 | 13:26 < jkczyz> sosthene: Good find! It uses BlockConnected and ChainStateFlushed to write to on-disk indexes
91 | 13:27 < sosthene> I see
92 | 13:28 < jkczyz> The remaining uses are not as important as these, but they are still interesting to look at
93 | 13:28 < sosthene> I see it also mentioned in the transaction.h file
94 | 13:28 < sosthene> https://github.com/bitcoin/bitcoin/blob/be0e8b4bff88b421128239e7140fc6bfdb654806/src/node/transaction.h
95 | 13:28 < jkczyz> submitblock RPC, ZMQ notificaitons, withing tests.
96 | 13:29 < lightlike> sosthene: my understanding is that the validation routines are called from the networking part (net_processing.cpp), and if we are sent something invalid, validationinterface will never be involved.
97 | 13:30 < sosthene> lightlike: I don't quite understand sorry, how do we know we have been sent something invalid without involving validationinterface?
98 | 13:33 < jkczyz> I may have been mistaken earlier about validating peer messages. Net processing will call the validation layer. Then only if the validation succeeds do subscrivers get notified
99 | 13:33 < lightlike> sosthene: we have a network thread that receives the messages. net_processing deals with the logic around these, and if we get a new block or tx, calls procedures in validation. Only if these procedures pass, validation will create signals for validationinterface that are passed on to the wallet etc.
100 | 13:34 < jkczyz> lightlike: yes, put more eloquently :)
101 | 13:34 < sosthene> lightlike: ok, it makes more sense, thanks!
102 | 13:34 < jkczyz> Alright, let's move on
103 | 13:34 < jkczyz> What is a CBlockLocator? How is it used in this PR? How else is it used?
104 | 13:37 < lightlike> jkczyz: i have encountered it in networking/getblocks messages, where it is used to determine the point up to which the chains of two peers are identical (for example in IBD when one peer has only part of the chain).
105 | 13:38 < sosthene> I found this, it makes sense with what lightlike says https://github.com/bitcoin/bitcoin/blob/78dae8caccd82cfbfd76557f1fb7d7557c7b5edb/src/primitives/block.h
106 | 13:39 < jkczyz> lightlike: yes, also in getheaders
107 | 13:39 < lightlike> https://github.com/bitcoin/bitcoin/blob/78dae8caccd82cfbfd76557f1fb7d7557c7b5edb/src/chain.cpp#L23 shows the logic behind it, how the internal vector of block hashes is filled
108 | 13:39 < jkczyz> See details: https://en.bitcoin.it/wiki/Protocol_documentation#getblocks
109 | 13:40 < jkczyz> lightlike: great! I was just about to ask how it was constructed
110 | 13:40 < jkczyz> Any insight into why it is constructed in that manner?
111 | 13:42 < lightlike> i think it's quite elegant, with the exponential thinning out: most of the time, two peers are just a few blocks away, so all the recent blocks are there. in rarer cases, where the chains differ strongly, multiple locators need to be exchanged to get the highest common block.
112 | 13:43 < jkczyz> lightlike: good observation! This is a nice way to limit the amount of data exchanged between peers.
113 | 13:43 < lightlike> jkczyz: do you know if satoshi came up with it?
114 | 13:44 < jkczyz> The locators are also used when writing to the on-disk block index
115 | 13:45 < jkczyz> lightlike: I'm afraid that predates my involvement. :) Checking out Github's blame layer might help determine the origin, though you may have to do some code archeology as the code has probably been moved from other files
116 | 13:46 < jkczyz> Back to one of the original questions: How is CBlockLocator used in this PR?
117 | 13:47 < lightlike> just to get the first element (which is the tip)
118 | 13:48 < sosthene> I see it in ChainStateFlushed, but I don't understand what it does
119 | 13:48 < jkczyz> lightlike: yup, this is logged as part of ChainStateFlushed.
120 | 13:50 < sosthene> jkczyz: I got to go, I'll read the transcript to get the end of the story, thank you and see you!
121 | 13:50 < jkczyz> sosthene: My understanding is that this is written to the on-disk index. I may be mistaken so others may correct me, but I believe this will give a compact path of block back to the genessis block
122 | 13:50 < lightlike> looks like everyone is at Scaling Bitcoin today :-)
123 | 13:51 < jkczyz> sosthene: thanks for joining!
124 | 13:51 < jkczyz> lightlike: indeed :)
125 | 13:51 < jkczyz> Hopefully there are some lurkers following along, too
126 | 13:52 < jkczyz> ok, let's finish up the last couple questions
127 | 13:52 < jkczyz> What refactorings are included in this PR?
128 | 13:53 < jkczyz> Hint: see the commits from the PR
129 | 13:54 < lightlike> MempoolEntryRemoved is renamed to TransactionRemovedFromMempool, some String functions are changed to return a const ref
130 | 13:54 < jkczyz> yep, fairly trivial changes but good for code hygiene
131 | 13:55 < jkczyz> Lastly, as we are running out of time: How can this PR be tested? Could any other tests have been added?
132 | 13:56 < lightlike> not sure about that one. I think there is some mechanism to check for log messages in functional tests.
133 | 13:57 < jkczyz> lightlike: yeah, I remember seeing that in the PR that logged bitcoind's args
134 | 13:58 < jkczyz> Here, I manually verified the logging through the p2p_compactblocks.py functional test
135 | 13:59 < lightlike> though i am not sure if testing log messages is a bit overkill.
136 | 13:59 < jkczyz> This was the flaky test from issue
137 | 13:59 < jkczyz> issue #12978
138 | 13:59 < jkczyz> https://github.com/bitcoin/bitcoin/issues/12978
139 | 13:59 < jkczyz> lightlike: agreed
140 | 14:00 < jkczyz> Other testing may be unit tests for newly added code
141 | 14:00 < jkczyz> And that's the bell!
142 | 14:00 < jkczyz> Thanks lightlike and sosthene for participating today!
143 | 14:00 < lightlike> by the way you don't have to force push for a new travis run. you could ask in #bitcoin-core-dev, several people can restart it.
144 | 14:00 < lightlike> thanks jkczyz!
145 | 14:01 < jkczyz> As jnewbery would want, please added any interesting PR's to https://github.com/bitcoin-core-review-club/bitcoin-core-review-club.github.io/issues/14
146 | 14:01 < jkczyz> So they can be covered in future weeks
147 | 14:01 < jkczyz> lightlike: Thanks! This was fun :)
148 | 14:02 < lightlike> jkczyz: gonna review/ack on github in the next days.
149 | {% endirc %}
150 |
--------------------------------------------------------------------------------
/_posts/2020-12-23-rc-testing.md:
--------------------------------------------------------------------------------
1 | ---
2 | layout: pr
3 | date: 2020-12-23
4 | title: "Testing Bitcoin Core 0.21 Release Candidates"
5 | components: ["tests"]
6 | host: jarolrod
7 | status: past
8 | ---
9 |
10 | ## Notes
11 |
12 | - Major versions of Bitcoin Core are released every 6-8 months. See the [Life
13 | Cycle documentation](https://bitcoincore.org/en/lifecycle/) for full details.
14 |
15 | - When all of the PRs for a release have been merged, _Release Candidate 1_
16 | (rc1) is tagged. The rc is then tested. If any issues are found, fixes are
17 | merged into the branch and a new rc is tagged. This continues until no major
18 | issues are found in an rc, and that rc is then considered to be the final
19 | release version.
20 |
21 | - To ensure that users don't experience issues with the new software, it's
22 | essential that the rcs are thoroughly tested. This special review club
23 | meeting is for people who want to help with that vital review process.
24 |
25 | - This [Bitcoin Core Release Candidate Testing
26 | Guide](https://gist.github.com/jarolrod/6495932dc6b598ddef49e6374b348f4e) has
27 | a tutorial for testing the release candidate.
28 |
29 | ## Meeting Log
30 |
31 | {% irc %}
32 | 18:00 #startmeeting
33 | 18:00 It’s a Christmas Miracle! The Bitcoin Core PR Review Club has returned for a Special Edition on testing the upcoming 0.21 release
34 | 18:00 You have been enlisted to ensure the upcoming Bitcoin Core 0.21 Release is ready for production by going through some of its changes.
35 | 18:00 Here are your mission instructions: https://gist.github.com/jarolrod/6495932dc6b598ddef49e6374b348f4e
36 | 18:00 If you accept this mission, send a ‘hi’ in the chat. (Also so that we know who’s here)
37 | 18:00 hi
38 | 18:00 hi
39 | 18:00 hi
40 | 18:00 hi
41 | 18:01 Hi
42 | 18:01 hi
43 | 18:01 Is anyone here for the first time?
44 | 18:01 hi
45 | 18:01 hi
46 | 18:01 Yes
47 | 18:01 hello there anastasiia, welcome!
48 | 18:02 I have linked to the guide, does everyone have it open?
49 | 18:02 hi
50 | 18:02 Yes
51 | 18:03 Sure do
52 | 18:03 yes
53 | 18:03 Yes
54 | 18:03 yes
55 | 18:03 Great. For all who’ve enlisted, here is a a magnifying glass, notebook, and pen. Let’s start inspecting!
56 | 18:04 Those here for the first time: If any question pops up, just go right ahead and say it. You don’t need to ask in order to ask a question.
57 | 18:04 We will begin with the section titled ‘Preparation’
58 | 18:04 You have the option of grabbing the release candidate binary or compiling from source code.
59 | 18:05 we'll wait for everyone to get there copy of the release candidate
60 | 18:05 hi
61 | 18:06 hi
62 | 18:06 Any preference on the side of the core devs? I'm guessing the preference from the tester should be compile from source code
63 | 18:08 hey there fodediop and sishir, we're just getting started. We're grabbing the rc candidate from the testing guide
64 | 18:08 I downloaded the binary and checked the SHA256 hash of the binary against the gitian build.
65 | 18:09 compiling is preferred, but compiling can take some time depending on the machine. Those who are not comfortable with compiling should grab the binary
66 | 18:10 hi
67 | 18:10 emzy: 🎉
68 | 18:11 welcome felixweis
69 | 18:11 emzy: did you use https://github.com/fanquake/core-review/tree/master/gitian-building to check that or the docs on bitcoin core?
70 | 18:13 ajonas: I have still an old setup for the gitian builds. It is this https://github.com/bitcoin-core/docs/blob/master/gitian-building.md
71 | 18:14 hi
72 | 18:14 By "old" you mean it shouldn't be followed anymore?
73 | 18:14 For the macOS binary, how do we specify using the /tmp/21-rc-test data directory on launch?
74 | 18:14 is the apple signing bug for rc3 tracked somewhere?
75 | 18:15 jonatack: welcome!
76 | 18:15 michaelfolkson: istm the goal is to test the binaries, as that is what most people will use...happy to be corrected on that
77 | 18:16 jarolrod: thanks ✨
78 | 18:17 michaelfolkson: good question. I think the docker one from fanquake seems to be easier to set up.
79 | 18:17 nevermind
80 | 18:18 does everyone have a copy of the current release candidate?
81 | 18:18 murtyjones: you can specify the data directory by passing -datadir=PATH on the command line or datadir=PATH in the bitcoin.conf configuration filse
82 | 18:18 file
83 | 18:19 murtyjones: /Applications/Bitcoin-Qt.app/Contents/MacOS/Bitcoin-Qt -datadir=/tmp/21-rc-test
84 | 18:19 thanks!
85 | 18:22 The testing guide is divided into four missions: `Testing the Wallet`, `Testing Torv3`, `Signet`, and `Anchors`
86 | 18:22 As you go through each section, please bring up any questions or problems that arise
87 | 18:22 one thought, the guide says to test the GUI or the command line per your preference, but if you can it's good to test both, as there can be issues in one and not the other
88 | 18:25 To be inclusive, some people may not be too comfortable with the command line. If you are able to test both, then great!
89 | 18:26 In the .dmg for macOS is afaik only the GUI version included. So I only tested that.
90 | 18:29 emzy: AFAIK that is correct
91 | 18:30 the tor conf can be more minimal, e.g. no need to set onlynet=onion
92 | 18:31 jonatack: I think it is important so see that you can bootstrap from tor only.
93 | 18:31 nor proxy/bind either in some cases
94 | 18:32 I can still do the testing for 0.21 on testnet version right?
95 | 18:33 hmrawal: yep, the testnet chain is fine, and definitely try signet too
96 | 18:33 jonatack: sure
97 | 18:34 hmrawal: the tor part in the document adds some mainnet tor V3 onions. They will not work for testnet.
98 | 18:34 should it take a while to connect to peers using tor? my node has been stuck on "Connecting to peers" for a bit and nodes keep appearing and disappearing from the "Peers" window
99 | 18:34 emzy: that's true
100 | 18:34 signet does IBD much faster
101 | 18:34 signet starts up much faster too. my go-to for quick manual testing.
102 | 18:34 felixweis: because of smaller blocks ;)
103 | 18:35 murtyjones: yes connecting to tor can take some time, we've supplied some torv3 nodes in the bitcoin.conf to try to get around this
104 | 18:35 You can't see which connections are anchor in the GUI or CLI (getpeerinfo etc)?
105 | 18:35 (y)
106 | 18:36 btw. I'm running sxjbhmhob2xasx3vdsy5ke5j5jwecmh3ca4wbs7wf6sg4g2lm3mbszqd.onion
107 | 18:36 windows -> network traffic
108 | 18:36 michaelfolkson: the anchor connections will be the initial block-only peers. you can see them easily with ./src/bitcoin-cli -netinfo 4
109 | 18:37 not sure but how much size is required in signet
110 | 18:37 emzy: which net?
111 | 18:38 felixweis: mainnet
112 | 18:38 can you run a signet instance too please?
113 | 18:39 emzy: just connected to you
114 | 18:39 felixweis: hm. Maybe on another host. This box is busy with the mainnet :)
115 | 18:39 if anyone wants to connect to emzy, run this on the command line:
116 | 18:39 ./src/bitcoin-cli addnode sxjbhmhob2xasx3vdsy5ke5j5jwecmh3ca4wbs7wf6sg4g2lm3mbszqd.onion onetry
117 | 18:39 jonatack: I see one onion connection incomming.
118 | 18:40 emzy: the subversion should have @jon at the end
119 | 18:40 for those on the tor section, here's some extra-credit work
120 | 18:40 1. Running `bitcoind` with the `-debug=tor` configuration option lets you see useful info on the onion configuration
121 | 18:40 2. This release includes a dashboard that lets you see useful network information. Instead of `getpeerinfo`, one can run `-netinfo 4`
122 | 18:40 jonatack: yes: "70016/Satoshi:21.99.0(@jon)/"
123 | 18:40 thanks jonatack for the tips
124 | 18:41 yes, -debug=tor is handy for seeing in the debug log if your tor setup has any issues or is working ok
125 | 18:42 it's not too noisy logging-wise so i always use it
126 | 18:44 felixweis: signet: 4sk25djzsqqscnv4yfbbntiuo6xxbmddfweppmpc6p6wlt33p36iluyd.onion
127 | 18:45 Have to drop off but thanks jarolrod!
128 | 18:45 murtyjones: thanks for testing!
129 | 18:46 jarolrod: this idea and guide and the one by ajonas is great. taking action, testing PRs and releases, and reporting feedback are a valuable part of reviewing.
130 | 18:47 I have started signet, says connecting to peers . Is there some config that we should do beforehand ?
131 | 18:47 Presumably those are temporary Tor nodes you've put up emzy. Would be good to know of a persistent Signet Tor node when doing Tor testing
132 | 18:48 (outside of this session)
133 | 18:48 hmrawal: the main config is to enable signet in your bitcoin.conf, one note is that for many it takes some time to connect as it is a new network
134 | 18:48 emzy, thanks still trying to get tor working
135 | 18:48 i have 4 tor v3 onion peers in signet ATM
136 | 18:48 hmrawal: emzy put up a signet tor node in the chat, you can addnode it
137 | 18:49 michaelfolkson: I could move the onions to some server. If people are interested.
138 | 18:49 Persistent jonatack? Can I have one? :)
139 | 18:49 two testing 0.21.0 and two running master (0.21.99)
140 | 18:49 qlci54ryrj6ywxgki5vvf45fwtp3yhdxxftgnhczvo37qpc2abqcjoid.onion:38333
141 | 18:49 Thanks
142 | 18:49 v7ajjeirttkbnt32wpy3c6w3emwnfr3fkla7hpxcfokr3ysd3kqtzmqd.onion:38333
143 | 18:49 jonatack: thank you! ✨
144 | 18:49 those two are 0.21
145 | 18:49 hx3x3dhdxd5jbz5popeydcehuj5bpekztoya5b7ymvidlvgj6ean6pid.onion:38333
146 | 18:49 lktswnpsh3b7ctsvnf7xrjfbesho5l7i3fds2kwvmzufjv7mzgrgoiid.onion:38333
147 | 18:50 They are running on VPS?
148 | 18:50 those two are 0.21.99 (master), i think most of them are persistent, maybe run by kalle or aj
149 | 18:52 With < 10 min left, are people close to being able to leave their comment on #20555?
150 | 18:52 Okay, I need a little help. I am running bitcoin qt and in another tab in terminal I entered jonatack's terminal command to add emzy's node but I get an error saying I need to start bitcoind server
151 | 18:53 hmrawal: iirc you cannot use bitcoin-cli to talk to bitcoin-qt
152 | 18:53 hmrawal: try entering it on the GUI (bitcoin qt) console
153 | 18:54 as jonatack said
154 | 18:54 jarolrod: you can, if bitcoin-qt was started with -server option
155 | 18:54 or you an add to your conf and restart
156 | 18:54 sipa: thanks, didn't know that!
157 | 18:54 hmrawal: on the GUI you have to use the console window (in bitcoin-qt) and type "addnode 4sk25djzsqqscnv4yfbbntiuo6xxbmddfweppmpc6p6wlt33p36iluyd.onion onetry"
158 | 18:55 ^^ for a signet node
159 | 18:55 We're getting close to time everyone, 5 minutes!
160 | 18:56 Thanks everyone for testing, but the mission is not done!
161 | 18:57 please report back your findings here: https://github.com/bitcoin/bitcoin/issues/20555
162 | 18:57 got tor working, was :9150 for some reason
163 | 18:57 are there any peers to connect for later testing ?
164 | 18:57 felixweis: are you on windows?
165 | 18:57 felixweis: that's the port tor-browser is using.
166 | 18:57 macos tor brwoser
167 | 18:58 afaik Port 9150 is the default one of tor browser
168 | 18:58 hmrawal2008: you have mainnet tor v3 peers in the guide, and signet ones here in the discussion
169 | 18:58 On making a comment on the issue page: list your hardware and operating system, the rc version you tested, what you were able to test, and what you found while testing
170 | 18:58 so will these peers be available for later use also right ?
171 | 18:59 If everything went well, that's great! let us know
172 | 18:59 I will have mine up for a few weeks.
173 | 18:59 (i don't see any tor v3 peers in testnet ATM)
174 | 18:59 testnet is just too large for IBD :( signet all the way!
175 | 19:00 That's time everyone!, thanks for participating, happy holidays!
176 | 19:00 hmrawal2008: yes, worth trying to connect to them, emzy's mainnet ones are stable and i think the signet ones are stable, too
177 | 19:00 thanks jarolrod for hosting :)
178 | 19:01 #endmeeting
179 | 19:01 thanks jarolrod, great initiative 🍰
180 | 19:01 thanks jarolrod!
181 | 19:01 thanks jarolrod
182 | 19:01 cool thanks, thanks jarolrod, thanks everyone and merry christmas
183 | 19:01 thank you everyone! happy holidays and happy new year!
184 | 19:01 thanks jarolrod!
185 | 19:01 Thanks jarolrod for hosting.... Happy Holidays and Merry Christmas everyone
186 | 19:02 🎄
187 | 19:03 Thanks jarolrod for hosting.... Happy Holidays and Merry Christmas everyone. See you 2021.
188 | {% endirc %}
189 |
--------------------------------------------------------------------------------
/_posts/2021-09-15-#22677.md:
--------------------------------------------------------------------------------
1 | ---
2 | layout: pr
3 | date: 2021-09-15
4 | title: "Cut the validation <-> txmempool circular dependency"
5 | pr: 22677
6 | authors: [glozow]
7 | components: ["p2p", "validation", "mempool"]
8 | host: glozow
9 | status: past
10 | commit: ab3fef71e
11 | ---
12 |
13 | ## Notes
14 |
15 | * A [circular dependency](https://en.wikipedia.org/wiki/Circular_dependency) exists when two or more
16 | modules depend on each other. [PR#13695](https://github.com/bitcoin/bitcoin/pull/13695) added a
17 | linter which statically analyzes files to detect circular dependencies. At the time, a few already
18 | existed and were listed in
19 | [`EXPECTED_CIRCULAR_DEPENDENCIES`](https://github.com/bitcoin/bitcoin/blob/053a5fc7d912d597cd6dc7376b479420d1eae1c0/test/lint/lint-circular-dependencies.sh#L11).
20 |
21 | * One of the circular dependencies in the codebase is validation <-> txmempool. These two modules
22 | are each heavily depended on by other modules, so their interdependency creates a "domino effect"
23 | into the rest of the codebase. To experience this for yourself, compile (with ccache), make a small
24 | edit to protocol.h (where P2P protocol messages are defined), then run `make` again and watch it
25 | rebuild txmempool and everything mempool-related.
26 |
27 | * [PR#22677](https://github.com/bitcoin/bitcoin/pull/22677) removes this dependency by removing
28 | txmempool's dependency on validation. It creates a `GetFiltered()` function that allows
29 | validation and other clients to pass in an arbitrary predicate, iterates through all of its entries,
30 | applies the predicate function to each of them, and returns the set of filtered iterators to mempool
31 | entries. This allows validation to apply consensus rules to all of the mempool entries without
32 | access to the mempool's internals.
33 |
34 | ## Questions
35 |
36 | 1. Did you review the PR? [Concept ACK, approach ACK, tested ACK, or
37 | NACK](https://github.com/bitcoin/bitcoin/blob/master/CONTRIBUTING.md#peer-review)?
38 |
39 | 2. What is a circular dependency? Why should we avoid circular dependencies?
40 |
41 | 3. What functionality lives in validation? What functionality lives in txmempool?
42 |
43 | 4. Why should/shouldn't validation depend on txmempool? Why should/shouldn't txmempool depend on
44 | validation?
45 |
46 | 5. One of the functions that causes txmempool to rely on validation is the `check()` function. What
47 | checks does it perform, and how does this PR split the checks into two categories?
48 |
49 | 6. Another function that causes txmempool to rely on validation is the `removeForReorg()` function,
50 | which calls `CheckSequenceLocks()` and `CheckFinalTx()`. Why do we need to re-check sequence
51 | locks in a reorg? How is it possible for a coinbase spend to become premature in a reorg?
52 |
53 | 7. [Clang Thread Safety Analysis
54 | annotations](https://github.com/bitcoin/bitcoin/blob/92aad5303b9b96c46015156b5dc96b48e9e7bc76/doc/developer-notes.md#threads-and-synchronization)
55 | give compile-time warnings about potential race conditions. How do we let the compiler know whether
56 | `cs_main` is held
57 | [here](https://github.com/bitcoin-core-review-club/bitcoin/blob/pr22677/src/validation.cpp#L387)?
58 |
59 | 8. Do you have any suggestions for alternative approaches to removing this circular dependency?
60 |
61 | ## Meeting Log
62 |
63 | {% irc %}
64 | 18:00 #startmeeting
65 | 18:00 Hello friends! Welcome to PR Review Club! Feel free to say hi to let us know you're here
66 | 18:00 hi!
67 | 18:01 We're looking at PR#22677 today: "Cut the validation <-> txmempool circular dependency"
68 | 18:01 hi
69 | 18:01 PR: https://github.com/bitcoin/bitcoin/pull/22677
70 | 18:01 Notes: https://bitcoincore.reviews/22677
71 | 18:01 hi
72 | 18:02 did anyone get a chance to look at the notes or review the PR?
73 | 18:02 n
74 | 18:03 n
75 | 18:03 0.5y
76 | 18:03 no problem. we'll start conceptual: What is a circular dependency, and why should we avoid circular dependencies?
77 | 18:04 Two or more modules depend on each other
78 | 18:04 hi
79 | 18:04 pg2: indeed!
80 | 18:04 testing is much easier
81 | 18:04 and why are circular dependencies bad?
82 | 18:04 we can't seperately use one module individually if another is linked with it (they might recursively call each other)
83 | 18:05 sandipndev: right, it's much harder to isolate and test our code if it's tangled up with a bunch of other stuff
84 | 18:06 What functionality lives in the validation module? What lives in txmempool?
85 | 18:06 with circular dependencies present, it's harder to get a clear mental model
86 | 18:06 valiation.{h.cpp} update local knowledge of the current best chain and corresponding UTXO set, and process new blocks
87 | 18:06 txmempool.{h.cpp} store the actual transaction in the pool (class `CTxMemPool`), and metadata about the transactions (class `CTxMemPoolEntry`)
88 | 18:07 larryruane: yeah, i agree with that too!
89 | 18:08 Removing circular dependencies is desirable, but are there any downside or risks for this kind of refactoring? (e.g. could this break any downstream systems built on top of Bitcoin Core?)
90 | 18:09 pg2: good answer. i might add (for the context of this PR) that txmempool is a "dumb" data structure that shouldn't know any details about consensus rules
91 | 18:09 (that's my opnion though of course)
92 | 18:10 i see there are a few expected circular dependencies, since circular deps are bad, why are they even present and expected?
93 | 18:11 pg2: good question, I suppose forks of bitcoin core could have some trouble incorporating our changes if our architectures diverge
94 | 18:11 because the codebase is old, and works, so we don't want to throw it out
95 | 18:12 for context, the circular dependencies linter was added in https://github.com/bitcoin/bitcoin/pull/13695
96 | 18:13 listing them explicitly probably helps us see when they're removed and be warned if someone adds a new one
97 | 18:14 having a list of expected circular dependencies allows us to use the linter without ripping everything out
98 | 18:14 sandipndev: does that answer your question?
99 | 18:15 yes, absolutely!
100 | 18:15 cool! :)
101 | 18:15 So, in case it wasn't clear, we currently have a circular dependency between txmempool and validation.
102 | 18:16 There would theoretically be 2 ways to remove this: make txmempool not depend on validation anymore, or make validation not depend on txmempool anymore
103 | 18:16 Why should/shouldn’t validation depend on txmempool? Why should/shouldn’t txmempool depend on validation?
104 | 18:17 Validation should depend on txmempool, because we need to know what is in the mempool to validate new transactions
105 | 18:17 pg2: very logical answer, I agree :D
106 | 18:17 Txmempool shouldn't depend on validation, because the transactions in mempool are already validated (and therefore donot need to depend on anything in validation.{h.cpp}).
107 | 18:18 i also think an argument could be made for neither depending on the other - you don't really need to know about the existence of a mempool in order to apply validation rules to a transaction
108 | 18:19 glozow: thanks for your answers. after this PR, is txmempool "complete dumb", or "dumb enough"? Or there are places where it still knows about consensus rules (unnecessarily)?
109 | 18:20 pg2: ooh good question
110 | 18:21 since i've already started saying possibly controversial things... another thing the mempool probably doesn't need to be responsible for is the fee estimator
111 | 18:21 <_aj_> glozow: "neither depending on the other" -- isn't validation the *action* of accepting a block (which means the txs in the block should no longer be in the mempool since they're no longer valid on top of the new tip) ; the consensus rules themselves are in consensus/ and script/ and the like?
112 | 18:22 <_aj_> (also the action of accepting a tx into the mempool)
113 | 18:23 _aj_: ah true, i completely agree. I guess in general I mean that consensus rules themselves (i.e. consensus/ and script/) and the mempool are not conceptually related
114 | 18:24 but yes, the action of validating transactions would depend on both
115 | 18:26 OK next question. One of the functions that causes txmempool to rely on validation is the `check()` function. What checks does it perform, and how does this PR split the checks into two categories?
116 | 18:27 For every transaction input in mempool, `check()` checks if the input refers to any other mempool transactions, otherwise it checks if the previous transaction output is unspent.
117 | 18:28 I don't find anywhere in `check()` checks the sender has enough balance to cover the sum of output amounts. Is there such a check somewhere?
118 | 18:29 pg2: see `CheckTxInputs` in src/consensus/tx_verify
119 | 18:29 glozow: thanks
120 | 18:29 https://github.com/bitcoin/bitcoin/blob/2161a058552ac938f2079b311a2d12f5d1772d01/src/consensus/tx_verify.cpp#L201-L206
121 | 18:30 pg2: yes, that's one thing that `check()` does. what else?
122 | 18:30 (oops - the link I sent has the wrong line numbers. it should be a few lines above, the `bad-txns-in-belowout` check)
123 | 18:33 glozow: that's the only part where I finished reading the code, that's why I can only give a partial answer
124 | 18:33 no worries. just seeing if anyone is willing to answer the rest :)
125 | 18:34 so I've (arbitrarily) split the `check()` assertions into 2 categories: contextual checks and internal consistency checks. Contextual is what pg2 already mentioned - we go through and make sure all the transactions' inputs refer to something available in our mempool or UTXO set.
126 | 18:36 Internal consistency checks are things like making sure the ancestor/descendant counts of each entry add up correctly, checking that our total fee accounting and dynamic memory usage are accurate, etc.
127 | 18:36 It may or may not make sense to split `check()` up this way; up to you as a reviewer
128 | 18:37 Moving on to next question: Another function that causes txmempool to rely on validation is the `removeForReorg()` function, which calls `CheckSequenceLocks()` and `CheckFinalTx()`. Why do we need to re-check sequence locks in a reorg?
129 | 18:38 Yes, I think this conceptual distinction between contextual and non-contextual checks appears also with respect to blocks
130 | 18:39 glozow: because time is "backing up" so a tx that used to be valid is no longer valid (yet)
131 | 18:40 larryruane: right! but... how is it possible for a coinbase spend in the mempool to become premature in a reorg? Don't we only reorg when we see a longer chain?!
132 | 18:41 a reorg could result in a shorter chain, because it may have more _work_
133 | 18:41 larryruane: aha!!!!
134 | 18:43 <_aj_> the first block in a multi-block reorg will be lower than the last block before the reorg even if difficulty doesn't change? [A B C D] -> [A E F G H], E has lower height than D?
135 | 18:44 <_aj_> (or are we only readding txs after processing all the reorg blocks somewhere?)
136 | 18:45 _aj_: i believe so
137 | 18:46 _aj_: righto. i think in a multi-block reorg we fill up a disconnectpool and then call `removeForReorg` after
138 | 18:46 not 100% sure tho?
139 | 18:47 will check
140 | 18:51 next question is about lock annotations. what are we saying to the compiler when we annotate a callable with `EXCLUSIVE_LOCKS_REQUIRED`?
141 | 18:55 <_aj_> glozow: (hmmmm, i think i had a bug way back whenever when i was investigating something related to this then!)
142 | 18:56 _aj_: (i am... about 16% confident in that statement)
143 | 18:58 hok so the answer to my previous question about lock annotations, the answer is here: https://clang.llvm.org/docs/ThreadSafetyAnalysis.html#requires-requires-shared
144 | 18:59 the rest of the questions are left as an exercise to the reader
145 | 18:59 thanks for coming!
146 | 18:59 #endmeeting
147 | {% endirc %}
148 |
--------------------------------------------------------------------------------
/_posts/2023-04-12-#27255-2.md:
--------------------------------------------------------------------------------
1 | ---
2 | layout: pr
3 | date: 2023-04-12
4 | title: "MiniTapscript: port Miniscript to Tapscript (Part 2)"
5 | pr: 27255
6 | authors: [darosior]
7 | components: ["descriptors"]
8 | host: josibake
9 | status: past
10 | commit: 6e3b37b
11 | ---
12 |
13 | ## Notes
14 |
15 | See notes from the [first part of this review club](/27255).
16 |
17 | ## Questions
18 |
19 | 1. Did you review the PR? [Concept ACK, approach ACK, tested ACK, or NACK](https://github.com/bitcoin/bitcoin/blob/master/CONTRIBUTING.md#peer-review)?
20 | What was your review approach?
21 |
22 | 1. In Miniscript, we have type modifiers, which guarantee additional properties for an expression. [866284d](https://github.com/bitcoin-core-review-club/bitcoin/commit/866284d007993551f681809d9e48175a3b0fe0c1) makes the wrapper "**d:**" have the "**u**" property under Tapscript.
23 | * What is the "**d:**" wrapper and the "**u**" type modifier?
24 | * Why is it that we can make **d:** have the **u** property here? Why not in non-Tapscript Miniscript?
25 |
26 | 1. This PR adds some logic for statically ensuring no spending path exceeds the stack size at execution time:
27 | * Why does this matter for Tapscript?
28 | * What's the approach taken by this PR? What are the pros/cons? (hint: [efdd154](https://github.com/bitcoin-core-review-club/bitcoin/commit/efdd1543597aff49c56a1abaa75b574be3b330db))
29 | * Can you think of an alternative approach to ensure no spending path exceeds the stack size?
30 |
31 | 1. In [e81635c](https://github.com/bitcoin-core-review-club/bitcoin/commit/e81635c39d99a158629544fefd765b3994f3d7c4), the scripts are optionally padded during fuzzing. Why?
32 |
33 | 1. What is the most significant change to the descriptor logic in this PR (hint: [08db38a](https://github.com/bitcoin-core-review-club/bitcoin/commit/08db38aca2fe9169b39507d928c1094be2116ad4)). Why is it needed?
34 |
35 |
36 | ## Meeting Log
37 |
38 | {% irc %}
39 | 17:00 < josie> #startmeeting
40 | 17:00 < josie> hi!
41 | 17:02 < kevkevin> hi
42 | 17:02 < josie> planning to review https://bitcoincore.reviews/27255-2 , which is a follow up to last weeks PR review club: https://bitcoincore.reviews/27255
43 | 17:03 < josie> although, it's awfully quiet in here...
44 | 17:03 < abubakr> hi
45 | 17:03 < glozow> hi
46 | 17:05 < brunoerg> hi
47 | 17:05 < josie> i recognize most of the names from last time, but I'll ask any way: did everyone here get a chance to attend part 1?
48 | 17:05 < josie> and if so, have you had a chance to review the questions for part 2?
49 | 17:06 < abubakar> yes
50 | 17:06 < kevkevin> yup was in part 1 didnt get too much of a chance to look at part 2's questions
51 | 17:08 < LarryRuane> Hi
52 | 17:09 < josie> cool, let's start with question 2: In Miniscript, we have type modifiers, which guarantee additional properties for an expression. commit 866284d makes the wrapper “d:” have the “u” property under Tapscript
53 | 17:10 < josie> first off, what are the "d:" wrapper and "u" fragment?
54 | 17:11 < abubakar> from miniscript website "d" is a dissatisfiable: does not include a signature or a hash preimage cannot rely on timelocks for being satisfied
55 | 17:12 < stickies-v> hi
56 | 17:12 < abubakar> "u" when satisfied the expression will have 1 on the stack
57 | 17:13 < josie> also - I typed that incorrectly: that should be "what is the 'u' type modifier"
58 | 17:13 < stickies-v> abubakar: perhaps confusingly, there's a difference between `d` the property and `d:` the wrapper
59 | 17:13 < stickies-v> as in - they're entirely unrelated
60 | 17:14 < abubakar> stickies-v: ohthanks
61 | 17:15 < josie> abubakar: I made the same mistake when writing the questions! it is confusingly named
62 | 17:15 < stickies-v> properties/type modifiers describe attributes of fragments (expressions), e.g. the `z` property says that a fragment with that property always consumes exactly 0 stack elements
63 | 17:15 < stickies-v> wrappers however take a fragment and add some script before/after it
64 | 17:16 < josie> stickies-v: that's a great explanation on the difference between wrappers and type modifiers, thanks!
65 | 17:17 < stickies-v> the `d:X` wrapper takes fragment X, and adds `DUP IF` before and `ENDIF` after the fragment, so you end up with `DUP IF [X] ENDIF`
66 | 17:19 < abubakar> stickies-v: thanks
67 | 17:19 < josie> what about the type modifier, "u"?
68 | 17:20 < stickies-v> fragments with type modifier `u` always put `1` on the stack when satisfied
69 | 17:20 < abubakar> i think its the one I reffered to earlier
70 | 17:20 < stickies-v> ah yes exactly you did!
71 | 17:21 < josie> abubakar: sorry, i missed this, but yes! when satisfied, expressions with the u property leave exactly 1 on the stack
72 | 17:22 < josie> so leading into the second part: why is okay for us to say the d:X fragment has the u property under tapscript?
73 | 17:22 < abubakar> It is called unit because of the 1 it leaves right?
74 | 17:23 < josie> abubakar: that's how I understood it: unit as in exactly 1
75 | 17:25 < abubakar> josie: okay
76 | 17:26 < stickies-v> before taproot, OP_IF would evaluate to true for any value > 0, whereas in taproot we require the argument to be either 0/1. Which means that in case of true, the argument is definitely 1. Because the `d:` wrapper duplicates the argument (OP_DUP), we can state that in case of satisfaction this fragment will put 1 on the stack
77 | 17:27 < josie> stickies-v: "OP_IF would evaluate true for any value > 0" .. was this true for _any_ transaction?
78 | 17:29 < stickies-v> hmmmm, i thought so but it sounds like i'm missing something?
79 | 17:30 < josie> perhaps a better way to phrase my question: was there anything pre-tapscript that would have prevented "regular" users from constructing transactions that used this OP_IF behavior?
80 | 17:31 < instagibbs> MINIMALIF was a standardness check for p2wsh iirc
81 | 17:32 < josie> instagibbs: bingo! the main problem here is miners are not bound by standardness, only validity
82 | 17:33 < josie> I don't have a link to it, but darosier has shown some nasty examples where miners could exploit this to steal funds in scripts that use thresholds
83 | 17:34 < josie> so if people remember from last week, MINIMALIF was added as a consensus rule in tapscript, which means it is now safe to say that d:X fragments can have the u property
84 | 17:34 < josie> moving on to question 3: "This PR adds some logic for statically ensuring no spending path exceeds the stack size at execution time"
85 | 17:35 < josie> why does this matter for tapscript?
86 | 17:36 < kevkevin> didn't CHECKMULTISIG have a limit of 20, and CHECKSIGADD removed that limit so now we're bound by the stack size limit, not sure if I'm remembering correctly
87 | 17:37 < instagibbs> miniscript does more than 20 signatures fine I think. BIP342 introduces "It is extended to also apply to the size of initial stack" which means it can't exceed limits ever and succeed. Not sure that answers it
88 | 17:37 < instagibbs> see https://github.com/bitcoin/bips/blob/master/bip-0342.mediawiki#resource-limits
89 | 17:37 < abubakar> because someone might use a script that add elements that are larger than stack size, standarness were lifted
90 | 17:39 < stickies-v> miniscript also checks if a script is sane, which (amongst others) also means that it is consensus valid and standard - this helps a lot with analysis of scripts. for example, you may not care about the entire script, just that you have a certain spending path to which you control the keys. but if it then turns out that that spending path is actually not spendable (because of stack size limit), that's a problem
91 | 17:39 < josie> kevkevin: CHECKMULTISIG did have a limit, but I think its more correct to say CHECKMULTISIG was disabled in tapscript and CHECKSIGADD was added, and CHECKSIGADD doesn't have a limit (aside from the stack size limit)
92 | 17:41 < kevkevin> josie: ok ya that makes more sense, thanks
93 | 17:41 < josie> stickies-v: great point re: miniscript. I think tapscript's decision to remove some of the "arbitrary" limits from before makes being able to reason about sanity much more straightforward
94 | 17:43 < josie> moving on the the second part of the question: "What’s the approach taken by this PR? What are the pros/cons?"
95 | 17:44 < josie> by approach, we are referring to how this PR ensures no spending path exceeds the stack size limit at execution time
96 | 17:50 < josie> for this one, the commit message is pretty helpful
97 | 17:53 < josie> haha the silence is pretty deafening on this one (which admittedly is a pretty in the weeds question), so lets finish with the last question about descriptors
98 | 17:53 < josie> What is the most significant change to the descriptor logic in this PR?
99 | 17:54 < stickies-v> sorry josie I didn't review this far ahead so just looking at the commit now - if you wouldn't mind summarizing your answer i'd appreciate that?
100 | 17:54 < stickies-v> on the approach and pros/cons
101 | 17:58 < josie> sure! basically, it tracks how the stack will be affected after the script executes, and also how many items will be pushed on the stack during execution. the second part is the part I don't fully understand, but it also checks the maximum witness size for a fragment and takes the lesser. regarding pros and cons, it mentions this being a conservative approach because the max
102 | 17:58 < josie> witness might not always happen along with the max stack size
103 | 17:59 < stickies-v> i think we also distinguish between satisfactions and dissatisfactions, right?
104 | 17:59 < josie> as a con, you could say since this is a conservative approximation, which might not let you do a script that you would otherwise be able to do with more precise accounting
105 | 18:00 < josie> stickies-v: I believe so, but this where my knowledge of script execution gets fuzzy
106 | 18:01 < josie> we'll stop here. thanks everyone for attending part 2! this is a really dense topic so I appreciate everyone muscling through
107 | 18:01 < josie> #endmeeting
108 | 20:58 @josie kevkevin Arguably, CHECKMULTISIG has no more or no less a limit (20) than CHECKSIGADD has a limit (1).
109 | 20:59 The point is that multi maps to just one CHECKMULTISIG, and thus inherits the 20 limit.
110 | 20:59 While multi_a maps to many CHECKSIGADD opcodes, so it is not bound to its 1 limit.
111 | 21:00 We could arguably define a multi() fragment that intelligently maps to multiple CHECKMULTISIG opcodes, and that wouldn't be bound by the 20 limit either, but that's just hiding a lot of complexity that can equally well be captured at a higher level (forcing the miniscript to combine multiple multi() fragments explicitly).
112 | {% endirc %}
113 |
--------------------------------------------------------------------------------
/_posts/2023-05-24-v25-rc-testing.md:
--------------------------------------------------------------------------------
1 | ---
2 | layout: pr
3 | date: 2023-05-24
4 | title: "Testing Bitcoin Core 25.0 Release Candidates"
5 | components: ["tests"]
6 | host: ismaelsadeeq
7 | status: past
8 | commit:
9 | ---
10 |
11 | ## Notes
12 |
13 | - Major versions of Bitcoin Core are released every 6-8 months. See the [Life
14 | Cycle documentation](https://bitcoincore.org/en/lifecycle/) for full details.
15 | - When all of the PRs for a release have been merged, _Release Candidate 1_
16 | (rc1) is tagged. The rc is then tested. If any issues are found, fixes are
17 | merged into the branch and a new rc is tagged. This continues until no major
18 | issues are found in an rc, and that rc is then considered to be the final
19 | release version.
20 | - To ensure that users don't experience issues with the new software, it's
21 | essential that the rcs are thoroughly tested. This special review club
22 | meeting is for people who want to help with that vital review process.
23 |
24 | - This [Bitcoin Core Release Candidate Testing
25 | Guide](https://github.com/bitcoin-core/bitcoin-devwiki/wiki/25.0-Release-Candidate-Testing-Guide) provides guidance for testing the release candidate.
26 |
27 | - It is recommended to go through the "Preparation" steps ahead of the meeting, especially if you want to compile from source. Verify and confirm the successful installation of v25rc2.
28 |
29 | - The testing guide relies on the tools `jq` which are not installed by default on each platform. For example on macOS, you can install these ahead of time using `brew install`. Alternatively, you can also modify the instructions to avoid using these tools as they are not strictly necessary and/or can be replaced by other tools.
30 |
31 | - For some of the tests, you might want to have the [signet](https://en.bitcoin.it/wiki/Signet) chain fully synced beforehand, so that you can just copy the signet directory into `/tmp/25-rc-test/` every time you run a test in a fresh environment.
32 |
33 | - The guide is just to get you started on testing, so feel free to read the [Release Notes](https://github.com/bitcoin-core/bitcoin-devwiki/wiki/25.0-Release-Notes-Draft)
34 | and bring ideas of other things you'd like to test!
35 |
36 | ## Meeting Log
37 |
38 | {% irc %}
39 | 16:59 #startmeeting
40 | 16:59 hi
41 | 16:59 hello
42 | 16:59 Hello everyone welcome to this special edition of bitcoin core pr review club
43 | 16:59 yaay
44 | 17:00 Hi every1
45 | 17:00 hi all !
46 | 17:00 Feel free to say hello
47 | 17:00 Anyone joining in for the first time?
48 | 17:02 We will be testing the changes and new features in the upcoming release of bitcoin core version 25.0
49 | 17:03
7 | {{ content }}
8 |
9 |
7 |
14 | {{ page.title }}
8 | 12 |{{ content }}
13 |
21 |
69 | {{ page.title }} ({{components}})
22 | 23 | 32 | 33 | {% if page.pr %} 34 |35 | 36 | https://github.com/bitcoin/bitcoin/pull/{{ page.pr }} 37 | 38 |
39 | {% elsif page.link %} 40 |41 | 42 | {{ page.link }} 43 | 44 |
45 | {% endif %} 46 | 47 |48 | Host: 49 | {{ page.host }} 50 | 51 | {% unless authors == empty %} 52 | - 53 | 56 | {%- endunless -%} 57 |
58 | 59 | {% if page.status == "past" %} 60 | {% if page.commit %} 61 |The PR branch HEAD was {{ page.commit }} at the time of this review club meeting.
62 | {%- endif -%} 63 | {%- endif -%} 64 | 65 |
66 | {{ content }} 67 |
68 || #{lineno} | " \ 75 | "#{time} | " \ 76 | "" \ 77 | "#{nick}" \ 78 | "#{message}" \ 79 | " | " \ 80 | "