├── .github └── workflows │ ├── build.yml │ └── publish.yml ├── .gitignore ├── 01_overview-and-development.adoc ├── 02_architecture.adoc ├── 03_consensus-and-validation.adoc ├── 04_wallet.adoc ├── 05_gui.adoc ├── 06_p2p.adoc ├── 07_mempool.adoc ├── 08_script.adoc ├── 09_build-system.adoc ├── 10_rpc.adoc ├── 404.html ├── Containerfile ├── Dockerfile ├── Gemfile ├── Gemfile.lock ├── Makefile ├── README.adoc ├── _config.yml ├── _includes ├── best-way.md └── head_custom.html ├── addition-removal-mempool.adoc ├── address-relay.adoc ├── addrman.adoc ├── all_chapters.adoc ├── appendix.adoc ├── architecture.adoc ├── asciidoc_workflow.adoc ├── backports.adoc ├── bips.adoc ├── bitcoin-design.adoc ├── block-relay.adoc ├── bootstrapping.adoc ├── build.adoc ├── building.adoc ├── calculating-balance.adoc ├── codedocs.adoc ├── coin-selection.adoc ├── comms.adoc ├── components-overview.adoc ├── consensus-and-validation.adoc ├── consensus-bugs.adoc ├── consensus-changes.adoc ├── consensus-exercises.adoc ├── consensus-libraries.adoc ├── consensus-model.adoc ├── consensus-specification.adoc ├── consensus-v-policy.adoc ├── constructing-transactions.adoc ├── contributing.adoc ├── contributor-exercises.adoc ├── contributors.adoc ├── controlling-the-wallet.adoc ├── css └── asciidoc.css ├── cwallet.adoc ├── debugging.adoc ├── deep-arch.adoc ├── design-principles.adoc ├── docinfo-footer.html ├── docinfo.html ├── entries ├── bugs.md └── first-contribution.md ├── exercise-intro.adoc ├── exercise_base.ipynb ├── exercise_tutorial.ipynb ├── fork-wishlist.adoc ├── github.adoc ├── glossary.adoc ├── gui-building.adoc ├── gui-initialization.adoc ├── gui-motivation.adoc ├── gui.adoc ├── hardcoded-consensus-values.adoc ├── images ├── compilation_firewall_amiti.jpg ├── jupyter_duplicate.png ├── pimpl_peerman_amiti.png └── pimpl_txrequest_amiti.png ├── index.adoc ├── index.html ├── index_epub.adoc ├── index_pdf.adoc ├── input-checks.adoc ├── jekyll_workflow.md ├── library-structure.adoc ├── links-onepage.adoc ├── links-separate.adoc ├── locating-consensus-code.adoc ├── manfunc.adoc ├── mempool-lifecycle.adoc ├── mempool-purpose.adoc ├── mempool-terminology.adoc ├── mempool-tx-format.adoc ├── mempool-unbroadcast.adoc ├── mempool.adoc ├── mermaid-config-pdf.json ├── mermaid-config.json ├── message-relay.adoc ├── multiple-chains.adoc ├── multiple-transactions.adoc ├── multiwallet.adoc ├── netgroupmanager.adoc ├── network-entropy.adoc ├── node-components.adoc ├── overview.adoc ├── p2p-attacks.adoc ├── p2p-design-philosophy.adoc ├── p2p-encryption.adoc ├── p2p-exercises.adoc ├── p2p-violations.adoc ├── p2p.adoc ├── package-relay.adoc ├── past-changes.adoc ├── peer-state.adoc ├── pinning-attacks.adoc ├── pr-maturation.adoc ├── puppeteer-config.json ├── qml-gui.adoc ├── reading.adoc ├── relay-preferences.adoc ├── reproducible.adoc ├── requirements.txt ├── responsible-disclosure.adoc ├── reviewing.adoc ├── robots.txt ├── rpc.adoc ├── script-appendix.adoc ├── script-cli.adoc ├── script.adoc ├── scriptpubkeymanagers.adoc ├── settings.adoc ├── single-transactions.adoc ├── source-organization.adoc ├── starting-ideas.adoc ├── stats.adoc ├── subtrees.adoc ├── testing-p2p-changes.adoc ├── testing-qt.adoc ├── testing.adoc ├── testnets.adoc ├── tests-overview.adoc ├── threads.adoc ├── transaction-creation.adoc ├── transaction-identification.adoc ├── transaction-relay.adoc ├── transaction-signing.adoc ├── transaction-tracking.adoc ├── transaction-validation.adoc ├── transactions-from-blocks.adoc ├── userspace-files.adoc ├── validating-scripts.adoc ├── wallet-database.adoc ├── wallet-encryption.adoc ├── wallet-exercises.adoc ├── wallet-init.adoc ├── wallet-interfaces.adoc ├── wallet-keys.adoc ├── wallet-locks.adoc └── wallet.adoc /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | # see https://github.com/actions/starter-workflows/blob/main/pages/jekyll.yml for reference 2 | name: Build 3 | on: 4 | pull_request: 5 | branches: [main, master] 6 | # Allows you to run this workflow manually from the Actions tab 7 | workflow_dispatch: 8 | concurrency: 9 | group: github-pages-build 10 | cancel-in-progress: false 11 | jobs: 12 | build-jekyll: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Checkout 16 | uses: actions/checkout@v4 17 | - name: Configure Pages 18 | id: pages 19 | uses: actions/configure-pages@v5 20 | - name: Install Ruby and Jekyll 21 | uses: ruby/setup-ruby@v1 22 | with: 23 | ruby-version: '3.2.2' 24 | bundler-cache: true 25 | cache-version: 0 26 | - name: Setup Node.JS 18 27 | uses: actions/setup-node@v4 28 | with: 29 | node-version: 18 30 | - name: Install mermaid-cli 31 | run: npm install -g @mermaid-js/mermaid-cli 32 | - name: Generate Site 33 | run: | 34 | bundle 35 | make all 36 | env: 37 | JEKYLL_ENV: production 38 | ADD_JEKYLL_ARGS: --baseurl "${{ steps.pages.outputs.base_path }}" 39 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | # see https://github.com/actions/starter-workflows/blob/main/pages/jekyll.yml for reference 2 | name: Publish to GitHub Pages 3 | on: 4 | push: 5 | branches: [master] 6 | # Allows you to run this workflow manually from the Actions tab 7 | workflow_dispatch: 8 | concurrency: 9 | group: github-pages-publish 10 | cancel-in-progress: false 11 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 12 | permissions: 13 | contents: read 14 | pages: write 15 | id-token: write 16 | jobs: 17 | build-jekyll: 18 | runs-on: ubuntu-latest 19 | environment: 20 | name: github-pages 21 | url: ${{ steps.deployment.outputs.page_url }} 22 | steps: 23 | - name: Checkout 24 | uses: actions/checkout@v4 25 | - name: Configure Pages 26 | id: pages 27 | uses: actions/configure-pages@v5 28 | - name: Install Ruby and Jekyll 29 | uses: ruby/setup-ruby@v1 30 | with: 31 | ruby-version: '3.2.2' 32 | bundler-cache: true 33 | cache-version: 0 34 | - name: Setup Node.JS 18 35 | uses: actions/setup-node@v4 36 | with: 37 | node-version: 18 38 | - name: Install mermaid-cli 39 | run: npm install -g @mermaid-js/mermaid-cli 40 | - name: Generate Site 41 | run: | 42 | bundle 43 | make publish 44 | env: 45 | JEKYLL_ENV: production 46 | ADD_JEKYLL_ARGS: --baseurl "${{ steps.pages.outputs.base_path }}" 47 | - name: Upload Artifacts 48 | uses: actions/upload-pages-artifact@v3 49 | deploy: 50 | runs-on: ubuntu-latest 51 | needs: build-jekyll 52 | environment: 53 | name: github-pages 54 | url: ${{steps.deployment.outputs.page_url}} 55 | steps: 56 | - name: Deploy to GitHub Pages 57 | id: deployment 58 | uses: actions/deploy-pages@v4 59 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | index.html 2 | .ipynb_checkpoints 3 | *Copy*.ipynb 4 | _site 5 | .sass-cache 6 | .jekyll-cache 7 | .jekyll-metadata 8 | vendor 9 | images 10 | -------------------------------------------------------------------------------- /01_overview-and-development.adoc: -------------------------------------------------------------------------------- 1 | //= Overview & Development process 2 | 3 | TIP: Document originally written by https://github.com/willcl-ark[Will Clark]. Help maintain this document: https://github.com/chaincodelabs/onboarding-to-bitcoin-core[https://github.com/chaincodelabs/onboarding-to-bitcoin-core^] 4 | 5 | TIP: This section has been updated to Bitcoin Core @ https://github.com/bitcoin/bitcoin/tree/v23.0[v23.0^] 6 | 7 | include::reading.adoc[] 8 | 9 | include::github.adoc[] 10 | 11 | include::reviewing.adoc[] 12 | 13 | include::contributing.adoc[] 14 | 15 | include::testing.adoc[] 16 | 17 | include::debugging.adoc[] 18 | 19 | include::past-changes.adoc[] 20 | 21 | include::building.adoc[] 22 | 23 | include::codedocs.adoc[] 24 | 25 | include::testnets.adoc[] 26 | 27 | include::manfunc.adoc[] 28 | 29 | include::starting-ideas.adoc[] 30 | 31 | include::comms.adoc[] 32 | 33 | include::pr-maturation.adoc[] 34 | 35 | include::backports.adoc[] 36 | 37 | include::reproducible.adoc[] 38 | 39 | include::contributors.adoc[] 40 | 41 | include::bips.adoc[] 42 | 43 | include::stats.adoc[] 44 | 45 | include::contributor-exercises.adoc[] 46 | 47 | //// 48 | == Solo work 49 | 50 | :bip-extensions-mail: https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2021-September/019457.html 51 | :core-dev-08-26: https://www.erisian.com.au/bitcoin-core-dev/log-2021-08-26.html 52 | 53 | * Read lsilva01's https://github.com/chaincodelabs/bitcoin-core-onboarding/blob/main/1.0_bitcoin_core_architecture.asciidoc[1.0 Bitcoin Architecture]. Particularly sections: 54 | ** Executables 55 | ** https://github.com/chaincodelabs/bitcoin-core-onboarding/blob/main/1.1_regions.asciidoc[Regions] (and all sub-sections) 56 | 57 | TODO: Add questions on current architecture of Core 58 | 59 | //// 60 | -------------------------------------------------------------------------------- /02_architecture.adoc: -------------------------------------------------------------------------------- 1 | include::architecture.adoc[] 2 | 3 | include::design-principles.adoc[] 4 | 5 | include::components-overview.adoc[] 6 | 7 | include::tests-overview.adoc[] 8 | 9 | include::threads.adoc[] 10 | 11 | include::library-structure.adoc[] 12 | 13 | include::source-organization.adoc[] 14 | 15 | include::userspace-files.adoc[] 16 | 17 | include::deep-arch.adoc[] 18 | 19 | include::subtrees.adoc[] 20 | -------------------------------------------------------------------------------- /03_consensus-and-validation.adoc: -------------------------------------------------------------------------------- 1 | include::consensus-and-validation.adoc[] 2 | 3 | include::locating-consensus-code.adoc[] 4 | 5 | include::consensus-model.adoc[] 6 | 7 | include::consensus-v-policy.adoc[] 8 | 9 | include::consensus-bugs.adoc[] 10 | 11 | include::consensus-changes.adoc[] 12 | 13 | include::fork-wishlist.adoc[] 14 | 15 | include::consensus-specification.adoc[] 16 | 17 | include::consensus-libraries.adoc[] 18 | 19 | include::hardcoded-consensus-values.adoc[] 20 | 21 | include::transaction-validation.adoc[] 22 | 23 | include::single-transactions.adoc[] 24 | 25 | include::multiple-transactions.adoc[] 26 | 27 | include::input-checks.adoc[] 28 | 29 | include::transactions-from-blocks.adoc[] 30 | include::multiple-chains.adoc[] 31 | 32 | include::responsible-disclosure.adoc[] 33 | 34 | include::consensus-exercises.adoc[] 35 | 36 | // == Removed text 37 | // 38 | // The outline of the mechanism at work is that a node relaying a transaction can slightly modify the signature in a way which is still acceptable to the underlying OpenSSL module. 39 | // Once the signature has been changed, the transaction ID (hash) will also change. 40 | // If the modified transaction is then included in a block, before the original, the effect is that the sender will still see the outgoing transaction as "unconfirmed" in their wallet. 41 | // The sender wallet should however also see the accepted (modified) outgoing transaction, so their balance will be calculated correctly, only a "stuck doublespend" will pollute their wallet. 42 | // The receiver will not perceive anything unordinary, unless they were tracking the incoming payment using the txid as given to them by the sender. 43 | -------------------------------------------------------------------------------- /04_wallet.adoc: -------------------------------------------------------------------------------- 1 | include::wallet.adoc[] 2 | 3 | include::wallet-database.adoc[] 4 | 5 | include::wallet-encryption.adoc[] 6 | 7 | include::transaction-tracking.adoc[] 8 | 9 | include::calculating-balance.adoc[] 10 | 11 | include::coin-selection.adoc[] 12 | 13 | include::transaction-creation.adoc[] 14 | 15 | include::wallet-interfaces.adoc[] 16 | 17 | include::wallet-init.adoc[] 18 | 19 | include::wallet-locks.adoc[] 20 | 21 | include::controlling-the-wallet.adoc[] 22 | 23 | include::cwallet.adoc[] 24 | 25 | include::scriptpubkeymanagers.adoc[] 26 | 27 | include::wallet-keys.adoc[] 28 | 29 | include::transaction-identification.adoc[] 30 | 31 | include::constructing-transactions.adoc[] 32 | 33 | include::multiwallet.adoc[] 34 | 35 | include::wallet-exercises.adoc[] 36 | -------------------------------------------------------------------------------- /05_gui.adoc: -------------------------------------------------------------------------------- 1 | include::gui.adoc[] 2 | 3 | include::gui-motivation.adoc[] 4 | 5 | include::gui-building.adoc[] 6 | 7 | include::gui-initialization.adoc[] 8 | 9 | include::qml-gui.adoc[] 10 | 11 | include::bitcoin-design.adoc[] 12 | 13 | include::testing-qt.adoc[] 14 | -------------------------------------------------------------------------------- /09_build-system.adoc: -------------------------------------------------------------------------------- 1 | include::build.adoc[] 2 | -------------------------------------------------------------------------------- /10_rpc.adoc: -------------------------------------------------------------------------------- 1 | include::rpc.adoc[] 2 | -------------------------------------------------------------------------------- /404.html: -------------------------------------------------------------------------------- 1 | --- 2 | permalink: /404.html 3 | layout: default 4 | --- 5 | 6 | 19 | 20 |
21 |

404

22 | 23 |

Page not found :(

24 |

The requested page could not be found.

25 |
26 | -------------------------------------------------------------------------------- /Containerfile: -------------------------------------------------------------------------------- 1 | FROM quay.io/official-images/debian 2 | RUN bash -c "echo cache buster 22" 3 | RUN bash -c "apt update" 4 | RUN apt --yes upgrade 5 | RUN apt --yes install build-essential curl procps git 6 | RUN gpg --keyserver keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB 7 | RUN \curl -sSL https://get.rvm.io/ | bash -s stable 8 | #RUN /bin/bash -l -c ". /etc/profile.d/rvm.sh && rvm install 2.6.4 && rvm use 2.6.4" 9 | RUN bash -l -c "rvm install 3.2.2" 10 | RUN bash -l -c "rvm use 3.2.2" 11 | WORKDIR /srv/ 12 | ENV PATH /usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin 13 | RUN bash -l -c "gem install http_parser.rb -v '0.8.0' --source 'https://rubygems.org/'" 14 | RUN bash -l -c "gem install bundler --source 'https://rubygems.org/'" 15 | RUN bash -l -c "gem install bundle --source 'https://rubygems.org/'" 16 | RUN bash -c "echo cache buster 23" 17 | RUN bash -l -c "bundle install" 18 | 19 | RUN bash -l -c "\curl https://nodejs.org/dist/v18.17.1/node-v18.17.1-linux-x64.tar.xz > /usr/local/node.tar.xz" 20 | RUN bash -l -c "cd /usr/local ; tar xJf node.tar.xz --strip-components=1" 21 | RUN bash -l -c "npm install -g @mermaid-js/mermaid-cli" 22 | 23 | ENV LANG C.UTF-8 24 | ENV LC_ALL C.UTF-8 25 | ENTRYPOINT bash -l 26 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ruby:3.2.2-slim-bookworm 2 | 3 | RUN apt update && \ 4 | apt --yes upgrade && \ 5 | apt --yes install build-essential curl procps git \ 6 | fonts-liberation libasound2 \ 7 | libatk-bridge2.0-0 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 \ 8 | libfontconfig1 libgbm1 libgcc1 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 \ 9 | libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 \ 10 | libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 \ 11 | --no-install-recommends && \ 12 | rm -rf /var/lib/apt/lists/* 13 | 14 | ARG UID=1000 15 | ARG GID=1000 16 | 17 | WORKDIR /srv/ 18 | COPY Gemfile /srv/Gemfile 19 | 20 | RUN groupadd -g ${GID} docs && \ 21 | useradd -m -u ${UID} -g docs -s /bin/bash docs 22 | RUN chown -R docs:docs /srv/ 23 | USER docs 24 | 25 | RUN bash -l -c "echo 'export GEM_HOME=${HOME}/.gem' >> ${HOME}/.bashrc \ 26 | && echo 'export GEM_PATH=${HOME}/.gem' >> ${HOME}/.bashrc \ 27 | && source ~/.bashrc \ 28 | && bundle config set --local path ${HOME}/.gem \ 29 | && bundle install" 30 | 31 | RUN bash -l -c "curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash \ 32 | && export NVM_DIR=\"\$([ -z \"${XDG_CONFIG_HOME-}\" ] && printf %s \"${HOME}/.nvm\" || printf %s \"${XDG_CONFIG_HOME}/nvm\")\" \ 33 | && [ -s \"\$NVM_DIR/nvm.sh\" ] && \\. \"\$NVM_DIR/nvm.sh\" \ 34 | && echo 'export PATH=${PATH}:/srv/node_modules/.bin' >> ${HOME}/.bashrc \ 35 | && source ~/.bashrc \ 36 | && nvm install node \ 37 | && npm install @mermaid-js/mermaid-cli" 38 | 39 | ENV LANG C.UTF-8 40 | ENV LC_ALL C.UTF-8 41 | ENTRYPOINT ["bash", "-l"] 42 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | ## If you update the version here, also update it in .travis.yml, .ruby-version, 4 | ## and README.md. Then push your branch and make sure Travis supports that 5 | ## version. 6 | ruby '3.2.2' 7 | 8 | gem "asciidoctor", "~>2.0.20" 9 | gem "rouge" 10 | 11 | ## If you add a new Gem below, run `bundle install` to install it. 12 | group :development do 13 | gem "jekyll", "~> 4.3.2" 14 | gem "just-the-docs" 15 | gem 'jekyll-redirect-from' 16 | # Lock `http_parser.rb` gem to `v0.6.x` on JRuby builds since newer versions of the gem 17 | # do not have a Java counterpart. 18 | #gem "http_parser.rb", "~> 0.6.0", :platforms => [:jruby] 19 | end 20 | 21 | group :jekyll_plugins do 22 | gem "asciidoctor-diagram" 23 | gem "jekyll-feed", "~> 0.12" 24 | gem "jekyll-asciidoc", "~> 3.0.0" 25 | end 26 | 27 | group :asciidoc_plugins do 28 | gem "asciidoctor-epub3" 29 | gem "asciidoctor-pdf" 30 | end 31 | 32 | group :testing do 33 | gem 'html-proofer' 34 | gem 'mdl' 35 | gem 'json-schema' 36 | gem 'toml' 37 | end 38 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: test-before-build build-other-versions build test-after-build 2 | build-other-versions: html pdf epub 3 | publish: test-before-build build-other-versions build 4 | production: clean all production-test 5 | 6 | ## If we call git in our tests without using the --no-pager option 7 | ## or redirecting stdout to another command, fail unconditionally. 8 | ## This addresses an issue where `make` can't be run within an IDE because 9 | ## it tries to paginate the output, see: 10 | ## https://github.com/bitcoinops/bitcoinops.github.io/pull/494#discussion_r546376335 11 | export GIT_PAGER='_contrib/kill0' 12 | JEKYLL_FLAGS = --future --drafts --unpublished --incremental 13 | ## Needed for github actions to work properly 14 | SHELL=/bin/bash 15 | 16 | clean: 17 | bundle exec jekyll clean 18 | 19 | preview: 20 | ## Don't do a full rebuild (to save time), but always rebuild index pages. 21 | rm -f _site/index.html \ 22 | 23 | bundle exec jekyll serve --host 0.0.0.0 $(JEKYLL_FLAGS) 24 | 25 | build: 26 | @# Tiny sleep for when running concurrently to ensure output 27 | @# files aren't created before changed input files are marked 28 | @# for schema validation. 29 | @sleep 0.1 30 | bundle exec jekyll build $(JEKYLL_FLAGS) $(ADD_JEKYLL_ARGS) 31 | 32 | test-before-build: $(compatibility_validation) $(topic_validation) 33 | ## Check for Markdown formatting problems 34 | @ ## - MD009: trailing spaces (can lead to extraneous
tags 35 | bundle exec mdl -g -r MD009 . 36 | 37 | test-after-build: build 38 | ## Check for broken Markdown reference-style links that are displayed in text unchanged, e.g. [broken][broken link] 39 | ## Check for duplicate anchors 40 | ! find _site/ -name '*.html' | while read file ; do \ 41 | cat $$file \ 42 | | egrep -o "(id|name)=[\"'][^\"']*[\"']" \ 43 | | sed -E "s/^(id|name)=//; s/[\"']//g" \ 44 | | sort | uniq -d \ 45 | | sed "s|.*|Duplicate anchor in $$file: #&|" ; \ 46 | done | grep . 47 | 48 | ## Check for broken links 49 | bundle exec htmlproofer --disable-external --ignore-urls '/^\/bin/.*/' ./_site 50 | 51 | html: 52 | mkdir -p bin 53 | ## Single page HTML 54 | ## Set imagesdir to parent_dir/images to work with jekyll build 55 | bundle exec asciidoctor -r asciidoctor-diagram -o onboarding-to-bitcoin-core.html index.adoc 56 | ## Delete non-deterministic asciidoctor output 57 | sed -i '/^Last updated /d' onboarding-to-bitcoin-core.html 58 | mv onboarding-to-bitcoin-core.html bin/ 59 | cp -r images bin/ 60 | 61 | pdf: 62 | bundle exec asciidoctor -r asciidoctor-pdf -b pdf -r asciidoctor-diagram -o onboarding-to-bitcoin-core.pdf index_pdf.adoc 63 | mv onboarding-to-bitcoin-core.pdf bin/ 64 | 65 | epub: 66 | bundle exec asciidoctor -b epub3 -r asciidoctor-epub3 -r asciidoctor-diagram -o index.epub index_epub.adoc 67 | mv index.epub bin/onboarding-to-bitcoin-core.epub 68 | -------------------------------------------------------------------------------- /README.adoc: -------------------------------------------------------------------------------- 1 | = Onboarding to Bitcoin Core 2 | 3 | This documentation is currently hosted at https://bitcoincore.academy[bitcoincore.academy] however this location may be subject to change in the future. 4 | 5 | See https://github.com/chaincodelabs/onboarding-to-bitcoin-core/blob/master/asciidoc_workflow.adoc[asciidoc_workflow.adoc] 6 | and https://github.com/chaincodelabs/onboarding-to-bitcoin-core/blob/master/jekyll_workflow.md[jekyll_workflow.md] for tips on contributing and instructions for running locally. 7 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | # Welcome to Jekyll! 2 | # 3 | # This config file is meant for settings that affect your whole blog, values 4 | # which you are expected to set up once and rarely edit after that. If you find 5 | # yourself editing this file very often, consider using Jekyll's data files 6 | # feature for the data you need to update frequently. 7 | # 8 | # For technical reasons, this file is *NOT* reloaded automatically when you use 9 | # 'bundle exec jekyll serve'. If you change this file, please restart the server process. 10 | # 11 | # If you need help with YAML syntax, here are some quick references for you: 12 | # https://learn-the-web.algonquindesign.ca/topics/markdown-yaml-cheat-sheet/#yaml 13 | # https://learnxinyminutes.com/docs/yaml/ 14 | # 15 | # Site settings 16 | # These are used to personalize your new site. If you look in the HTML files, 17 | # you will see them accessed via {{ site.title }}, {{ site.email }}, and so on. 18 | # You can create any custom variable you would like, and they will be accessible 19 | # in the templates via {{ site.myvariable }}. 20 | 21 | title: Bitcoin Core Onboarding 22 | email: your-email@example.com 23 | description: >- # this means to ignore newlines until "baseurl:" 24 | Bitcoin Core developer onboarding documentation 25 | baseurl: "" # the subpath of your site, e.g. /blog 26 | url: "https://bitcoincore.academy" # the base hostname & protocol for your site, e.g. http://example.com 27 | twitter_username: chaincodelabs 28 | github_username: chaincodelabs 29 | 30 | # Build settings 31 | theme: just-the-docs 32 | # Color scheme supports "light" and "dark" 33 | color_scheme: dark 34 | plugins: 35 | - asciidoctor-diagram 36 | - jekyll-feed 37 | - jekyll-asciidoc 38 | 39 | asciidoc: 40 | processor: asciidoctor 41 | 42 | asciidoctor: 43 | attributes: 44 | imagesdir: /images 45 | mermaid-config: mermaid-config.json 46 | mermaid-format: svg 47 | mermaid-puppeteer-config: puppeteer-config.json 48 | source-highlighter: rouge 49 | rouge-style: github 50 | icons: font 51 | 52 | keep_files: 53 | - images 54 | 55 | # Footer "Edit this page on GitHub" link text 56 | gh_edit_link: true # show or hide edit this page link 57 | gh_edit_link_text: "Edit this page on GitHub." 58 | gh_edit_repository: "https://github.com/chaincodelabs/onboarding-to-bitcoin-core" # the github URL for your repo 59 | gh_edit_branch: "master" # the branch that your docs is served from 60 | gh_edit_view_mode: "edit" # "tree" or "edit" if you want the user to jump into the editor immediately 61 | 62 | mermaid: 63 | # Version of mermaid library 64 | # Pick an available version from https://cdn.jsdelivr.net/npm/mermaid/ 65 | version: "9.1.3" 66 | 67 | # Exclude from processing. 68 | # The following items will not be processed, by default. 69 | # Any item listed under the `exclude:` key here will be automatically added to 70 | # the internal "default list". 71 | # 72 | # Excluded items can be processed by explicitly listing the directories or 73 | # their entries' file path in the `include:` list. 74 | exclude: 75 | - .jekyll-cache/ 76 | - all_chapters.adoc 77 | - 01_overview-and-development.adoc 78 | - 02_architecture.adoc 79 | - 03_consensus-and-validation.adoc 80 | - 04_wallet.adoc 81 | - 05_gui.adoc 82 | - 06_p2p.adoc 83 | - 07_mempool.adoc 84 | - 08_script.adoc 85 | - 09_build-system.adoc 86 | - 10_rpc.adoc 87 | - asciidoc_workflow.adoc 88 | - index.adoc 89 | - index_pdf.adoc 90 | - index_epub.adoc 91 | - settings.adoc 92 | - README.adoc 93 | - book.html 94 | - Gemfile 95 | - Gemfile.lock 96 | - jekyll_workflow.md 97 | - Makefile 98 | - mermaid-config.json 99 | - node_modules/ 100 | - vendor/bundle/ 101 | - vendor/cache/ 102 | - vendor/gems/ 103 | - vendor/ruby/ 104 | -------------------------------------------------------------------------------- /_includes/best-way.md: -------------------------------------------------------------------------------- 1 | _Ensuring that Bitcoin Core remains highly secure software requires 2 | reviewers and maintainers scrutinize every contribution, no matter how 3 | minor it seems. The best way to avoid wasting the time of reviewers and 4 | risking having them ignore your contribution is to read the entire 5 | onboarding guide, as described on the [main page](/). However, if 6 | you're going to contribute anyway without reading all of the 7 | documentation, this quickstart will guide you to the most relevant parts 8 | of the contributor documentation._ 9 | -------------------------------------------------------------------------------- /_includes/head_custom.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 20 | -------------------------------------------------------------------------------- /address-relay.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Address relay 2 | :page-nav_order: 70 3 | :page-parent: P2P 4 | [[address_relay]] 5 | === Address relay 6 | 7 | The Bitcoin network uses `addr` messages to communicate (node) network addresses. 8 | See the https://en.bitcoin.it/wiki/Protocol_documentation#addr[Bitcoin wiki p2p documentation^] for more details. 9 | Good address propagation improves network connectivity and increases the difficulty of executing an eclipse attack. 10 | 11 | Bitcoin Core nodes will periodically self-announce (also known as self-advertise) their own network address to peers. 12 | When a Bitcoin Core node receives an `addr` message that contains 10 addresses or fewer, it forwards those addresses with a timestamp within 10 minutes of the current time to 1 or 2 peers, selected at random. 13 | If we assume all nodes do this, then self-announcements should reach a large portion of the nodes on the network. The timestamp condition is there to ensure that the relay of a given address stops after some time. 14 | 15 | Since https://github.com/bitcoin/bitcoin/pull/22387[PR#22387^], there is a rate limit for address relay processing, so that addresses from peers that send too many of them are ignored which can help to prevent CPU/memory exhaustion attacks. 16 | 17 | ==== `Addr` privacy 18 | 19 | For some time, it was possible for a spy node to easily scrape the full contents of any reachable node's `AddrMan`. 20 | The spy just had to connect to a victim node multiple times and execute `GETADDR`. 21 | This scraped data could then be used to infer private information about the victim. 22 | 23 | For example, a spy could monitor the victim's `AddrMan` content in real time and figure out which peers a node is connected to. 24 | A spy could also compare the `AddrMan` content from two different connections (e.g. one identified by Tor address and one identified by IPv4) and figure out that it's actually the same physical node (`fingerprinting`). 25 | 26 | https://github.com/bitcoin/bitcoin/pull/18991[PR#18991^] was a first step towards fixing these privacy issues. 27 | By limiting (caching) the leaked portion of `AddrMan`, these inference activities became much harder. 28 | Caching in this context means that the `ADDR` response (which is only a small subset of a node's `AddrMan` content) remains the same for every `GETADDR` call during (roughly) a day. 29 | 30 | ==== Addr black holes 31 | 32 | We know that some nodes on the network do _not_ relay `addr` messages that they receive. 33 | Two known cases are block-relay-only connections from Bitcoin Core nodes, and connections from certain light clients. 34 | We refer to these connections as `addr` black holes. 35 | `addr` messages go in, but they never escape! 36 | 37 | If a large portion of the connections on the network are `addr` black holes, then `addr` propagation may be negatively impacted: self-announcements might not reach a majority of nodes on the network in a timely fashion. 38 | It'd be better if we could somehow avoid picking black holes as the 1 or 2 peers that we select for relaying `addr` messages to. 39 | 40 | https://github.com/bitcoin/bitcoin/pull/21528[PR#21528^] defers initialization of `m_addr_known` of inbound peers until the peer sends an address related message (`addr`, `addrv2`, `getaddr` or `sendaddrv2`). 41 | The node uses the presence of `m_addr_known` to decide whether the peer is a candidate for relaying `addr` messages received from the network. 42 | 43 | ==== addrv2 44 | 45 | https://github.com/bitcoin/bitcoin/pull/19031[PR#19031^] is a proposed implementation of the https://github.com/bitcoin/bips/blob/9286b5254317d9e73fb25c5f0acd2b2d9937843e/bip-0155.mediawiki[BIP155^] `addrv2` message, a new P2P message format proposed in early 2019 by Wladimir J. van der Laan to gossip longer node addresses. 46 | 47 | The `addrv2` message is required to support https://trac.torproject.org/projects/tor/wiki/doc/NextGenOnions[next-generation Tor v3 Onion addresses^], the https://geti2p.net[Invisible Internet Project (I2P)^], and potentially other networks that have longer endpoint addresses than fit in the 128 bits/16 bytes of the current `addr` message. 48 | 49 | -------------------------------------------------------------------------------- /all_chapters.adoc: -------------------------------------------------------------------------------- 1 | include::01_overview-and-development.adoc[] 2 | 3 | include::02_architecture.adoc[] 4 | 5 | include::03_consensus-and-validation.adoc[] 6 | 7 | include::04_wallet.adoc[] 8 | 9 | include::05_gui.adoc[] 10 | 11 | include::06_p2p.adoc[] 12 | 13 | include::07_mempool.adoc[] 14 | 15 | include::08_script.adoc[] 16 | 17 | include::09_build-system.adoc[] 18 | 19 | include::10_rpc.adoc[] 20 | 21 | include::appendix.adoc[] 22 | 23 | include::glossary.adoc[] 24 | -------------------------------------------------------------------------------- /appendix.adoc: -------------------------------------------------------------------------------- 1 | = Appendix 2 | :page-nav_order: 100 3 | 4 | [id=pimpl-technique] 5 | == PIMPL technique 6 | 7 | :pimpl_addrman_review: https://bitcoincore.reviews/22950 8 | 9 | The Bitcoin Core codebase contains many classes of the form `class *Impl`. 10 | These classes are taking advantage of the Pointer to Implementation https://en.cppreference.com/w/cpp/language/pimpl[technique^] which helps to both provide more stable ABIs and also to reduce compile-time dependencies. 11 | 12 | .Some of the current Bitcoin Core PIMPL classes 13 | [source,cpp] 14 | ---- 15 | AddrManImpl 16 | ChainImpl 17 | NodeImpl 18 | PeerManagerImpl 19 | WalletImpl 20 | 21 | FieldImpl 22 | DBImpl 23 | ExternalSignerImpl 24 | NotificationsHandlerImpl 25 | RPCHandlerImpl 26 | IpcImpl 27 | ProcessImpl 28 | RPCMethodImpl 29 | SketchImpl 30 | DescriptorImpl 31 | ---- 32 | 33 | Amiti Uttarwar {pimpl_addrman_review}[hosted^] a PR review club "Pimpl AddrMan to abstract implementation details" which contains information on the design aims, advantages and disadvantages. 34 | Below are copies of the annotated pictures she created and included to assist learning. 35 | 36 | .PIMPL peerman 37 | [link=https://bitcoincore.reviews/22950] 38 | image::pimpl_peerman_amiti.png[] 39 | 40 | .PIMPL txrequest 41 | [link=https://bitcoincore.reviews/22950] 42 | image::pimpl_txrequest_amiti.png[] 43 | 44 | .Compilation firewall 45 | [link=https://bitcoincore.reviews/22950] 46 | image::compilation_firewall_amiti.jpg[] 47 | 48 | -------------------------------------------------------------------------------- /architecture.adoc: -------------------------------------------------------------------------------- 1 | = Architecture 2 | :page-nav_order: 10 3 | :page-has_children: true 4 | 5 | TIP: This section has been updated to Bitcoin Core @ https://github.com/bitcoin/bitcoin/tree/v23.0[v23.0^] 6 | 7 | Bitcoin Core v0.1 contained 26 source code and header files (*.h/cpp) with _main.cpp_ containing the majority of the business logic. 8 | As of v23.0 there are more than 800 source files (excluding _bench/_, _test/_ and all subtrees), more than 200 benchmarking and cpp unit tests, and more than 200 python tests and lints. 9 | -------------------------------------------------------------------------------- /asciidoc_workflow.adoc: -------------------------------------------------------------------------------- 1 | = Use of the asciidoc format 2 | 3 | The https://asciidoctor.org/[asciidoc] format has been used for this documentation. 4 | 5 | To use asciidoc format, ensure that the `asciidoctor` tool is installed (and on your system $PATH). 6 | Also, it can be useful to ensure your editor understands the filetype, e.g. for Vim you can install https://github.com/habamax/vim-asciidoctor[vim-asciidoctor]. 7 | 8 | A quick reference for asciidoc formatting can be found https://docs.asciidoctor.org/asciidoc/latest/syntax-quick-reference/[here]. 9 | 10 | This documentation follows the asciidoc recommended practice of https://asciidoctor.org/docs/asciidoc-recommended-practices/#one-sentence-per-line[One Sentence Per Line]. 11 | If you choose to contribute (thank you!) please try and continue this practice to make future editing more straightforward. 12 | 13 | == Generating _index.html_ for static web hosting 14 | 15 | You can generate an _index.html_ file for static web hosting by installing the <> tool and then running: 16 | 17 | [source,bash] 18 | ---- 19 | asciidoctor index.adoc 20 | ---- 21 | 22 | Simply host _index.html_ along with the _images/_ directory and its contents at the root of your web server to serve the documentation. 23 | 24 | === Adding mermaid diagrams 25 | 26 | If you add a mermaid diagram to an article, you must add: 27 | 28 | ---- 29 | :mermaid-puppeteer-config: ./puppeteer-config.json 30 | ---- 31 | 32 | To the header of the file, in order that the Container/Dockerfile can have mermaid-cli pass the puppeteer config to puppeteer. 33 | 34 | === Regenerate mermaid diagrams 35 | 36 | In order to regenerate the mermaid diagrams, you will need https://github.com/mermaid-js/mermaid-cli[mermaid-cli] and https://docs.asciidoctor.org/diagram-extension/latest/[asciidoctor-diagram] installed. 37 | 38 | With both of the above programs on your PATH, you can then run command: 39 | 40 | [source,shell] 41 | ---- 42 | asciidoctor -r asciidoctor-diagram index.adoc 43 | ---- 44 | 45 | To generate index.html, along with required images in the `/images` sub-directory. 46 | 47 | === Update _index.html_ automatically as a pre-commit hook 48 | 49 | If you prefer you can write a simple pre-commit hook to update _index.html_ file automatically at each commit. 50 | An example of this might be: 51 | 52 | ._$PROJECT_ROOT/.git/hooks/pre-commit_ 53 | [source,bash] 54 | ---- 55 | #!/usr/bin/bash 56 | 57 | asciidoctor index.adoc 58 | ---- 59 | -------------------------------------------------------------------------------- /backports.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Backports 2 | :page-nav_order: 140 3 | :page-parent: Overview and Development Process 4 | === Backports 5 | 6 | Bitcoin Core often backports fixes for bugs and soft fork activations into previous software releases. 7 | 8 | Generally maintainers will handle backporting for you, unless for some reason the process will be too difficult. 9 | If this point is reached a decision will be made on whether the backport is abandoned, or a specific (new) fix is created for the older software version. 10 | 11 | == Software Life-cycle 12 | 13 | An overview of the software life-cycle for Bitcoin Core can be found at https://bitcoincore.org/en/lifecycle/ 14 | 15 | -------------------------------------------------------------------------------- /bips.adoc: -------------------------------------------------------------------------------- 1 | :page-title: BIPs 2 | :page-nav_order: 180 3 | :page-parent: Overview and Development Process 4 | [[bips]] 5 | == BIPs 6 | 7 | Bitcoin uses Bitcoin Improvement Proposals (BIPs) as a design document for introducing new features or behaviour into bitcoin. 8 | Bitcoin Magazine describes what a BIP is in their article https://bitcoinmagazine.com/guides/what-is-a-bitcoin-improvement-proposal-bip[What Is A Bitcoin Improvement Proposal (BIP)^], specifically highlighting how BIPs are not necessarily binding documents required to achieve consensus. 9 | 10 | The BIPs are currently hosted on GitHub in the bitcoin/bips https://github.com/bitcoin/bips[repo^]. 11 | 12 | [TIP] 13 | .BIP process 14 | ==== 15 | The BIPs include https://github.com/bitcoin/bips/tree/master/bip-0002.mediawiki[BIP 2^] which self-describes the BIP process in more detail. 16 | Of particular interest might be the sections https://github.com/bitcoin/bips/tree/master/bip-0002.mediawiki#BIP_types[BIP Types^] and https://github.com/bitcoin/bips/tree/master/bip-0002.mediawiki#BIP_workflow[BIP Workflow^]. 17 | ==== 18 | 19 | === What does having a BIP number assigned to an idea mean 20 | 21 | Bitcoin Core https://github.com/bitcoin/bitcoin/pull/22665[issue #22665^] described how BIP125 was not being strictly adhered to by Bitcoin Core. 22 | This raised discussion amongst developers about whether the code (i.e. "the implementation") or the BIP itself should act as the specification, with most developers expressing that they felt that "the code was the spec" and any BIP generated was merely a design document to aid with re-implementation by others, and should be corrected if necessary. 23 | 24 | NOTE: This view was not completely unanimous in the community. 25 | 26 | For consensus-critical code most Bitcoin Core Developers consider "the code is the spec" to be the ultimate source of truth, which is one of the reasons that recommending running other full node implementations can be so difficult. 27 | A knock-on effect of this was that there were calls for review on BIP2 itself, with respect to how BIPs should be updated/amended. 28 | Newly-appointed BIP maintainer Karl-Johan Alm (a.k.a. kallewoof) posted his thoughts on this to the https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2021-September/019457.html[bitcoin-dev mailing list^]. 29 | 30 | In summary a BIP represents a design document which should assist others in implementing a specific feature in a compatible way. 31 | These features are optional to usage of Bitcoin, and therefore implementation of BIPs are not required to use Bitcoin, only to remain compatible. 32 | Simply being _assigned_ a BIP does *not* mean that an idea is endorsed as being a "good" idea, only that it is fully-specified in a way that others could use to re-implement. 33 | Many ideas are assigned a BIP and then never implemented or used on the wider network. 34 | -------------------------------------------------------------------------------- /bitcoin-design.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Bitcoin design 2 | :page-nav_order: 30 3 | :page-parent: GUI 4 | == Bitcoin design 5 | 6 | The https://bitcoin.design/guide/[Bitcoin design guide^] provides some guidance on common pitfalls that Bitcoin GUI designers should look out for when designing apps (like `bitcoin-qt`). 7 | 8 | -------------------------------------------------------------------------------- /bootstrapping.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Bootstrapping 2 | :page-nav_order: 50 3 | :page-parent: P2P 4 | [[bootstrapping]] 5 | == Bootstrapping 6 | 7 | Bootstrapping is probably the most dangerous moment in a node's life. 8 | If the new node cannot make at least one connection to an honest node, from whom it can eventually learn more honest addresses, then it may not ever be able to join the most-work bitcoin chain without manual user intervention. 9 | 10 | NOTE: Manual intervention here would require the user to find the IP address of a known-honest node and connect to it either using `addnode` or `connect`. 11 | 12 | When the node first starts up, and if no node addresses are manually specified, we have no choice but to fetch addresses from one (or more) hardcoded DNS seed(s) the list of which can be found in https://github.com/bitcoin/bitcoin/blob/v23.0/src/chainparams.cpp#L121-L129[_src/chainparams.cpp_^]. 13 | 14 | If the node is fed only attacker-controlled addresses by one or more dishonest DNS seed(s) then it has little opportunity to join the rest of the honest network. 15 | However, if one or more of the addresses returned by the DNS query are honest then we want the node to be able to (eventually) find and connect to the honest network. 16 | 17 | Note that if the DNS seed queries are unsuccessful, or the node is being run in a Tor-only mode (and currently the DNS seeds cannot support long Tor V3 addresses) then bitcoind will fall back to connecting to a hard-coded https://github.com/bitcoin/bitcoin/blob/v23.0/src/chainparamsseeds.h[list^] of seed nodes. 18 | This fall back functionality could help to protect against e.g. an attack on the DNS seed infrastructure. 19 | 20 | [[service_flags]] 21 | == Service flags 22 | 23 | Nodes can advertise https://github.com/bitcoin/bitcoin/blob/v24.0.1/src/protocol.h#L266-L296[service flags] (a.k.a. "service bits") indicating which services that node supports. 24 | 25 | == Managing connections 26 | 27 | An enumeration of the different types of connections, along with detailed descriptions on their functions, can be found in https://github.com/bitcoin/bitcoin/blob/v23.0/src/net.h#L117-L184[src/_net.h_]. 28 | 29 | -------------------------------------------------------------------------------- /build.adoc: -------------------------------------------------------------------------------- 1 | = Build system 2 | :page-nav_order: 80 3 | 4 | [source,cpp] 5 | ---- 6 | return Error("Build system section not implemented yet!"); 7 | ---- 8 | 9 | -------------------------------------------------------------------------------- /building.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Building Bitcoin Core 2 | :page-nav_order: 70 3 | :page-parent: Overview and Development Process 4 | === Building from source 5 | 6 | When building Bitcoin Core from source, there are some platform-dependant instructions to follow. 7 | 8 | To learn how to build for your platform, visit the Bitcoin Core https://github.com/bitcoin/bitcoin/tree/master/doc[bitcoin/doc^] directory, and read the file named "build-\*.md", where "*" is the name of your platform. 9 | For windows this is "build-windows.md", for macOS this is "build-osx.md" and for most linux distributions this is "build-unix.md". 10 | 11 | There is also a guide by Jon Atack on how to https://jonatack.github.io/articles/how-to-compile-bitcoin-core-and-run-the-tests[compile and test Bitcoin Core^]. 12 | 13 | Finally, Blockchain Commons also offer a guide to https://github.com/BlockchainCommons/Learning-Bitcoin-from-the-Command-Line/blob/master/A2_0_Compiling_Bitcoin_from_Source.md[building from source^]. 14 | 15 | ==== Cleaner builds 16 | 17 | It can be helpful to use a separate build directory e.g. `build/` when compiling from source. 18 | This can help avoid spurious Linker errors without requiring you to run `make clean` often. 19 | 20 | From within your Bitcoin Core source directory you can run: 21 | 22 | [source,bash] 23 | ---- 24 | # Clean current source dir in case it was already configured 25 | make distclean 26 | 27 | # Make new build dir 28 | mkdir build && cd build 29 | 30 | # Run normal build sequence with amended path 31 | ../autogen.sh 32 | ../configure --your-normal-options-here 33 | make -j `nproc` 34 | make check 35 | ---- 36 | 37 | [TIP] 38 | ==== 39 | To run individual functional tests using the bitcoind binary built in an out-of-source build change directory back to the root source and specify the _config.ini_ file from within the build directory: 40 | 41 | [source,bash] 42 | ---- 43 | $ pwd 44 | /path/to/source/build 45 | $ cd .. 46 | $ test/functional/p2p_ping.py --configfile build/test/config.ini 47 | ---- 48 | ==== 49 | -------------------------------------------------------------------------------- /codedocs.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Codebase documentation 2 | :page-nav_order: 80 3 | :page-parent: Overview and Development Process 4 | === Codebase documentation 5 | 6 | Bitcoin Core uses https://www.doxygen.nl/index.html[Doxygen^] to generate developer documentation automatically from its annotated C++ codebase. 7 | Developers can access documentation of the current release of Bitcoin Core online at https://doxygen.bitcoincore.org/[doxygen.bitcoincore.org^], or alternatively can generate documentation for their current git `HEAD` using `make docs` (see https://github.com/bitcoin/bitcoin/tree/master/doc/developer-notes.md#generating-documentation[Generating Documentation^] for more info). 8 | -------------------------------------------------------------------------------- /coin-selection.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Coin selection 2 | :page-nav_order: 40 3 | :page-parent: Wallet 4 | include::links-onepage.adoc[] 5 | [id=coin-selection-overview] 6 | === Coin selection 7 | 8 | See https://bitcoinops.org/en/topics/coin-selection/[Bitcoin Optech^] for more information on coin selection. 9 | There is a section digging deeper into the coin selection code found xref:constructing-transactions.adoc#coin-selection[below]. 10 | To select inputs to a transaction our primary considerations are privacy and fees. 11 | 12 | The below sections form an overview of creating a transaction via `CreateTransactionInternal()`. 13 | 14 | ==== `AvailableCoins()` 15 | 16 | The gist of how we generate a list of coins available to spend (via `AvailableCoins()`) is that we iterate `mapWallet` and check for coins that: 17 | 18 | * Are not immature coinbase outputs 19 | * Are not conflicted 20 | * Must be at least in our mempool 21 | * Not currently replacing or being replaced by another transaction 22 | * Are not locked 23 | * Are `IsMine` 24 | * Are `spendable` 25 | 26 | ...and return them as a `std::vector`. 27 | 28 | [id=GroupOutputs] 29 | ==== `GroupOutputs()` 30 | 31 | Once we have this vector of coins `GroupOutputs()` will turn them into ``OutputGroup``s. 32 | An `OutputGroup` consists of outputs with the same script, i.e. "coins sent to the same address". 33 | 34 | //// 35 | Effective Value is = Coins value - Fee 36 | //// 37 | 38 | [id=selectCoins] 39 | ==== `selectCoins()` 40 | 41 | If you manually choose inputs, it will add outputs to the transaction automatically. 42 | It tries first to make sure that all outputs selected have 6 confirmations, if unsuccessful it then tries again with 1 confirmation as the lower bound. 43 | 44 | For change outputs it starts with 1 confirmation and then again with 0. 45 | If this is still unsuccessful it increases the number of ancestors and descendants that unconfirmed change can have. 46 | 47 | ==== `AttemptSelection()` 48 | 49 | This function is orchestrating the <> creation, and then the <>. 50 | Currently, this is always based on the xref:constructing-transactions.adoc#coin-selection[waste metric]. 51 | 52 | It is using 3 algorithms and then selecting the "best" of the three (based on the waste metric): 53 | 54 | . Branch n bound (bnb) 55 | . Knapsack 56 | . Single Random Draw (SRD) 57 | 58 | There is currently an idea that a limited SRD could replace Knapsack in the future. 59 | Due to this plan for removal, it would not make sense to focus development effort on improving the Knapsack algorithm at this time. 60 | -------------------------------------------------------------------------------- /comms.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Communication channels 2 | :page-nav_order: 120 3 | :page-parent: Overview and Development Process 4 | === #bitcoin-core-dev IRC channel 5 | 6 | The Bitcoin Core project has an IRC channel `#bitcoin-core-dev` available on the Libera.chat network. 7 | If you are unfamiliar with IRC there is a short guide on how to use it with a client called Matrix https://hackmd.io/ZcCoEDnOSTSqb2RDa7fB8Q[here^]. 8 | IRC clients for all platforms and many terminals are available. 9 | 10 | "Lurking" (watching but not talking) in the IRC channel can both be a great way to learn about new developments as well as observe how new technical changes and issues are described and thought about from other developers with an adversarial mindset. 11 | Once you are comfortable with the rules of the room and have questions about development then you can join in too! 12 | 13 | [NOTE] 14 | ==== 15 | This channel is reserved for discussion about _development of the Bitcoin Core software only_, so please don't ask general Bitcoin questions or talk about the price or other things which would be off topic in there. 16 | 17 | There are plenty of other channels on IRC where those topics can be discussed. 18 | ==== 19 | 20 | There are also regular meetings held on #bitcoin-core-dev which are free and open for anyone to attend. 21 | Details and timings of the various meetings are found https://bitcoincore.org/en/meetings/[here^]. 22 | 23 | === Communication 24 | 25 | In reality there are no hard rules on choosing a discussion forum, but in practice there are some common conventions which are generally followed: 26 | 27 | * If you want to discuss the codebase of the Bitcoin Core implementation, then discussion on either the GitHub repo or IRC channel is usually most-appropriate. 28 | * If you want to discuss changes to the core bitcoin protocol, then discussion on the mailing list is usually warranted to solicit feedback from (all) bitcoin developers, including the many of them that do not work on Bitcoin Core directly. 29 | ** If mailing list discussions seem to indicate interest for a proposal, then creation of a BIP usually follows. 30 | 31 | If discussing something Bitcoin Core-related, there can also be a question of whether it's best to open an Issue, to first discuss the problem and brainstorm possible solution approaches, or whether you should implement the changes as you see best first, open a PR, and then discuss changes in the PR. 32 | Again, there are no hard rules here, but general advice would be that for potentially-controversial subjects, it might be worth opening an Issue first, before (potentially) wasting time implementing a PR fix which is unlikely to be accepted. 33 | 34 | Regarding communication timelines it is important to remember that many contributors are unpaid volunteers, and even if they are sponsored or paid directly, nobody owes you their time. 35 | That being said, often during back-and-forth communication you might want to nudge somebody for a response and it's important that you do this in a courteous way. 36 | There are again no hard rules here, but it's often good practice to give somebody on the order of a few days to a week to respond. 37 | Remember that people have personal lives and often jobs outside of Bitcoin development. 38 | -------------------------------------------------------------------------------- /consensus-and-validation.adoc: -------------------------------------------------------------------------------- 1 | :page-nav_order: 20 2 | :page-has_children: true 3 | include::links-onepage.adoc[] 4 | = Consensus and Validation 5 | 6 | TIP: This section has been updated to Bitcoin Core @ https://github.com/bitcoin/bitcoin/tree/v23.0[v23.0^] 7 | 8 | One of the fundamental concepts underlying bitcoin is that nodes on the network are able to maintain decentralized consensus on the ordering of transactions in the system. 9 | 10 | The primary mechanism at work is that all nodes validate every block, and every transaction contained within that block, against their own copy of the consensus rules. 11 | The secondary mechanism is that in the event of a discrepancy between two competing chain tips nodes should follow the chain with the most cumulative proof-of-work. 12 | The result is that all honest nodes in the network will eventually converge onto a single, canonical, valid chain. 13 | 14 | WARNING: If all nodes do not compute consensus values identically (including edge cases) a chainsplit will result. 15 | 16 | For more information on how the bitcoin networks' decentralized consensus mechanism works see the Mastering Bitcoin section on https://github.com/bitcoinbook/bitcoinbook/tree/develop/ch10.asciidoc#decentralized-consensus[decentralized consensus^]. 17 | 18 | TIP: In Bitcoin Core there are an extra level of validation checks applied to incoming transactions in addition to consensus checks called "policy" which have a slightly different purpose, see xref:consensus-v-policy.adoc#consensus_vs_policy[consensus vs policy] for more information on the differences between the two. 19 | 20 | Consensus:: 21 | A collection of functions and variables which **must** be computed identically to all https://bitnodes.io/nodes/[other^] nodes on the network in order to remain in consensus and therefore on the main chain. 22 | 23 | Validation:: 24 | Validation of blocks, transactions and scripts, with a view to permitting them to be added to either the blockchain (must pass consensus checks) or our local mempool (must pass policy checks). 25 | -------------------------------------------------------------------------------- /consensus-libraries.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Consensus libraries 2 | :page-nav_order: 70 3 | :page-parent: Consensus and Validation 4 | == libbitcoinkernel 5 | 6 | The https://github.com/bitcoin/bitcoin/issues/24303[libbitcoinkernel^] project seeks to modularise Bitcoin Cores' consensus engine and make it easier for developers to reason about when they are modifying code which could be consensus-critical. 7 | 8 | This project differs from `libbitcoinconsensus` in that it is designed to be a stateful engine, with a view to eventually: being able to spawn its own threads, do caching (e.g. of script and signature verification), do its own I/O, and manage dynamic objects like a mempool. 9 | Another benefit of fully extracting the consensus engine in this way may be that it becomes easier to write and reason about consensus test cases. 10 | 11 | In the future, if a full de-coupling is successfully completed, other Bitcoin applications might be able to use `libbitcoinkernel` as their own consensus engine permitting multiple full node implementations to operate on the network in a somewhat safer manner than many of them operate under today. 12 | The initial objective of this library however is to actually have it used by Bitcoin Core internally, something which is not possible with libbitcoinconsensus due to it's lack of caching and state (making it too slow to use). 13 | 14 | Some examples have surfaced recently where script validation in the BTCD code (used internally by LND) has diverged from the results from Bitcoin Core: 15 | 16 | . https://twitter.com/brqgoo/status/1579216353780957185[Witness size check^]: https://github.com/lightningnetwork/lnd/issues/7002[issue^] and https://github.com/btcsuite/btcd/pull/1896[fix^] 17 | . https://twitter.com/brqgoo/status/1587397646125260802[Max witness items check^]: https://github.com/btcsuite/btcd/issues/1906[issue^] and https://github.com/btcsuite/btcd/pull/1907[fix^]. 18 | 19 | The implementation approaches of libbitcoinconsensus and libbitcoinkernel also differ; with lb-consensus parts of consensus were moved into the library piece by piece, with the eventual goal that it would be encapsulated. 20 | lb-kernel takes a different approach -- first cast a super wide net around everything needed to run a consensus engine, and then gradually strip pieces out where they can be. 21 | In theory this should get us something which Bitcoin Core can use much faster (in fact, you can build the optional `bitcoin-chainstate` binary which already has some functionality). 22 | 23 | Part of libbitcoinkernel has been merged in via Carl Dong's https://github.com/bitcoin/bitcoin/pull/24304[`bitcoin-chainstate` PR^]. 24 | It also has its own project https://github.com/bitcoin/bitcoin/projects/18[board^] to track progress. 25 | 26 | == libbitcoinconsensus 27 | 28 | The libbitcoinconsensus library has been deprecated since Bitcoin Core v27.0, with the Release Note: 29 | 30 | [quote] 31 | ____ 32 | libbitcoinconsensus is deprecated and will be removed for v28. This library has 33 | existed for nearly 10 years with very little known uptake or impact. It has 34 | become a maintenance burden. 35 | 36 | The underlying functionality does not change between versions, so any users of 37 | the library can continue to use the final release indefinitely, with the 38 | understanding that Taproot is its final consensus update. 39 | 40 | In the future, libbitcoinkernel will provide a much more useful API that is 41 | aware of the UTXO set, and therefore be able to fully validate transactions and 42 | blocks. (#29189) 43 | ____ 44 | -------------------------------------------------------------------------------- /consensus-model.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Consensus model 2 | :page-nav_order: 10 3 | :page-parent: Consensus and Validation 4 | === Consensus model 5 | 6 | The consensus model in the codebase can be thought of as a database of the current state of the blockchain. 7 | When a new block is learned about it is processed and the consensus code must determine which block is the current best. 8 | Consensus can be thought of as a function of available information -- it's output is simply a deterministic function of its input. 9 | 10 | There are a simple set of rules for determining the best block: 11 | 12 | . Only consider valid blocks 13 | . Where multiple chains exist choose the one with the most cumulative Proof of Work (PoW) 14 | . If there is a tie-breaker (same height and work), then use first-seen 15 | 16 | The result of these rules is a tree-like structure from genesis to the current day, building on only valid blocks. 17 | 18 | Whilst this is easy-enough to reason about in theory, the implementation doesn't work exactly like that. 19 | It must consider state, do I go forward or backwards for example. 20 | 21 | == Validation in Bitcoin Core 22 | 23 | Originally consensus and validation were much of the same thing, in the same source file. 24 | However splitting of the code into strongly delineated sections was never fully completed, so validation.* files still hold some consensus codepaths. 25 | 26 | -------------------------------------------------------------------------------- /consensus-specification.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Consensus specification 2 | :page-nav_order: 60 3 | :page-parent: Consensus and Validation 4 | include::links-onepage.adoc[] 5 | == Bitcoin core consensus specification 6 | 7 | A common question is where the bitcoin protocol is documented, i.e. specified. 8 | However bitcoin does not have a formal specification, even though many ideas have some specification (in xref:bips.adoc[BIPS]) to aid re-implementation. 9 | 10 | IMPORTANT: The requirements to be compliant with "the bitcoin spec" are to be bug-for-bug compatible with the Bitcoin Core implementation. 11 | 12 | The reasons for Bitcoin not having a codified specification are historical; Satoshi never released one. 13 | Instead, in true "Cypherpunks write code" style and after releasing a general whitepaper, they simply released the first client. 14 | This client existed on it's own for the best part of two years before others sought to re-implement the rule-set in other clients: 15 | 16 | * https://github.com/libbitcoin/libbitcoin-system/commit/9dea4682bf0e4247f3c4cb8a6c140ade61bf7df7[libbitcoin^] 17 | * https://github.com/bitcoinj/bitcoinj/commit/d1036b101f01b7ab79fc3e10e5199f80f478674d[BitcoinJ^] 18 | 19 | A forum https://bitcointalk.org/index.php?topic=195.msg1611#msg1611[post^] from Satoshi in June 2010 had however previously discouraged alternative implementations with the rationale: 20 | 21 | [quote,Satoshi Nakamoto] 22 | ____ 23 | ... 24 | 25 | I don't believe a second, compatible implementation of Bitcoin will ever be a good idea. So much of the design depends on all nodes getting exactly identical results in lockstep that a second implementation would be a menace to the network. The MIT license is compatible with all other licenses and commercial uses, so there is no need to rewrite it from a licensing standpoint. 26 | ____ 27 | 28 | It is still a point of contention amongst some developers in the community, however the fact remains that if you wish to remain in consensus with the majority of (Bitcoin Core) nodes on the network, you must be _exactly_ bug-for-bug compatible with Bitcoin Core's consensus code. 29 | 30 | TIP: If Satoshi _had_ launched Bitcoin by providing a specification, could it have ever been specified well-enough to enable us to have multiple node implementations? 31 | 32 | [TIP] 33 | ==== 34 | One mechanism often employed by those who want to run custom node software is to position an up-to-date Bitcoin Core node to act as a "gateway" to the network. 35 | Internally your own node can then make a single connection to this Bitcoin Core node. 36 | This means that your custom internal node will now only receive transactions and blocks which have passed Bitcoin Core's consensus (or policy) checks, allowing you to be sure that your custom node is not accepting objects which could cause you to split onto a different chain tip. 37 | ==== 38 | -------------------------------------------------------------------------------- /consensus-v-policy.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Consensus vs Policy 2 | :page-nav_order: 20 3 | :page-parent: Consensus and Validation 4 | [[consensus_vs_policy]] 5 | == Consensus vs Policy 6 | 7 | What is the difference between consensus and policy checks? 8 | Both seem to be related to validating transactions. 9 | We can learn a lot about the answer to this question from sdaftuar's StackExchange https://bitcoin.stackexchange.com/questions/100317/what-is-the-difference-between-policy-and-consensus-when-it-comes-to-a-bitcoin-c/100319#100319[answer^]. 10 | 11 | The answer teaches us that policy checks are a superset of validation checks -- that is to say that a transaction that passes policy checks has implicitly passed consensus checks too. 12 | Nodes perform policy-level checks on all transactions they learn about before adding them to their local mempool. 13 | Many of the policy checks contained in `policy` are called from inside `validation`, in the context of adding a new transaction to the mempool. 14 | 15 | -------------------------------------------------------------------------------- /contributing.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Contributing code 2 | :page-nav_order: 30 3 | :page-parent: Overview and Development Process 4 | === Contributing code 5 | 6 | This section details some of the processes surrounding code contributions to the Bitcoin Core project along with some common pitfalls and tips to try and avoid them. 7 | 8 | ==== Branches 9 | 10 | You should *not* use the built-in GitHub branch https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-and-deleting-branches-within-your-repository[creation^] process, as this interferes with and confuses the Bitcoin Core git process. 11 | 12 | Instead, you should use either the native https://git-scm.com/downloads[`git`^] or the GitHub https://github.com/cli/cli[`gh` cli^] (requires `git`) tools to create your own branches locally, before pushing them to your fork of the repo, and opening a PR against the Bitcoin Core repo from there. 13 | 14 | ==== Creating a PR 15 | 16 | Jon Atack's article https://jonatack.github.io/articles/how-to-contribute-pull-requests-to-bitcoin-core[How To Contribute Pull Requests To Bitcoin Core^] describes some less-obvious requirements that any PR you make might be subjected to during peer review, for example that it needs an accompanying test, or that an intermediate commit on the branch doesn't compile. 17 | It also describes the uncodified expectation that Contributors should not only be writing code, but perhaps more importantly be providing reviews on other Contributors' PRs. 18 | Most developers enjoy writing their own code more than reviewing code from others, but the decentralized review process is arguably the most critical defence Bitcoin development has against malicious actors and therefore important to try and uphold. 19 | 20 | IMPORTANT: Jon's estimates of "5-15 PR reviews|issues solved" per PR submitted is not a hard requirement, just what Jon himself feels would be best for the project. Don't be put off submitting a potentially valuable PR just because "you haven't done enough reviews"! 21 | 22 | For some tips on how to maintain an open PR using git, such as how to redo commit history, as well as edit specific commits, check out this https://github.com/satsie/bitcoin-notez/blob/master/bitcoin-core-development/git-guide.md[guide^]. 23 | 24 | ==== Commit messages 25 | 26 | When writing commit messages be sure to have read Chris Beams' "How to Write a Git Commit Message" https://chris.beams.io/posts/git-commit/[blog post^]. 27 | As described in CONTRIBUTING.md, PRs should be prefixed with the component or area the PR affects. 28 | Common areas are listed in CONTRIBUTING.md section: https://github.com/bitcoin/bitcoin/tree/master/CONTRIBUTING.md#creating-the-pull-request[Creating the pull request^]. 29 | Individual commit messages are also often given similar prefixes in the commit title depending on which area of the codebase the changes primarily affect. 30 | 31 | [TIP] 32 | ==== 33 | Unless there is a merge conflict (usually detected by DrahtBot), don't rebase your changes on master branch before pushing. 34 | If you avoid rebases on upstream, Github will show a very useful "Compare" button which reviewers can often use to quickly re-ACK the new changes if they are sufficiently small. 35 | If you _do_ rebase this button becomes useless, as all the rebased changes from master get included and so a full re-review may be needed. 36 | Developer review time is currently our major bottleneck in the project! 37 | ==== 38 | -------------------------------------------------------------------------------- /contributors.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Project roles 2 | :page-nav_order: 160 3 | :page-parent: Overview and Development Process 4 | == Organisation & roles 5 | 6 | The objective of the Bitcoin Core Organisation is to represent an entity that is decentralized as much as practically possible on a centralised platform. 7 | One where no single Contributor, Member, or Maintainer has unilateral control over what is/isn't merged into the project. 8 | Having multiple Maintainers, Members, Contributors, and Reviewers gives this objective the best chance of being realised. 9 | 10 | === Contributors 11 | 12 | Anyone who contributes code to the codebase is labelled a Contributor by GitHub and also by the community. 13 | As of Version 23.0 of Bitcoin Core, there are > 850 individual Contributors credited with changes. 14 | 15 | === Members 16 | 17 | Some Contributors are also labelled as Members of the https://github.com/orgs/bitcoin/people[Bitcoin organisation^]. 18 | There are no defined criteria for becoming a Member of the organisation; persons are usually nominated for addition or removal by current Maintainer/Member/Admin on an ad-hoc basis. 19 | Members are typically frequent Contributors/Reviewers and have good technical knowledge of the codebase. 20 | 21 | Some members also have some additional permissions over Contributors, such as adding/removing tags on issues and Pull Requests (PRs); however, being a Member **does not** permit you to merge PRs into the project. 22 | Members can also be assigned sections of the codebase in which they have specific expertise to be more easily requested for review as Suggested Reviewers by PR authors. 23 | 24 | //// 25 | NOTE: It is neither necessary nor desirable to "request reviews" from suggested reviewers in a normal workflow. Doing so without a good reason might be interpreted as being pushy and having the opposite result than intended. 26 | //// 27 | 28 | === Maintainers 29 | 30 | Some organisation Members are also project Maintainers. 31 | The number of maintainers is arbitrary and is subject to change as people join and leave the project, but has historically been less than 10. 32 | PRs can only be merged into the main project by Maintainers. 33 | While this might give the illusion that Maintainers are in control of the project, the Maintainers' role dictates that they *should not* be unilaterally deciding which PRs get merged and which don't. 34 | Instead, they should be determining the mergability of changes based primarily on the reviews and discussions of other Contributors on the GitHub PR. 35 | 36 | Working on that basis, the Maintainers' role becomes largely _janitorial_. 37 | They are simply executing the desires of the community review process, a community which is made up of a decentralized and diverse group of Contributors. 38 | 39 | //// 40 | A list of Maintainers and suggested Reviewers can be found in the https://github.com/bitcoin/bitcoin/tree/master/REVIEWERS[REVIEWERS^] document. 41 | As the document states, these are *not* the only people who should be reviewing PRs. 42 | The project needs as many reviews on each PR as possible, ideally from a diverse range of Reviewers. 43 | //// 44 | 45 | === Organisation fail-safes 46 | 47 | It is possible for a "rogue PR" to be submitted by a Contributor; we rely on systematic and thorough peer review to catch these. 48 | There has been https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2021-September/019490.html[discussion^] on the mailing list about purposefully submitting malicious PRs to test the robustness of this review process. 49 | 50 | In the event that a Maintainer goes rogue and starts merging controversial code, or conversely, _not_ merging changes that are desired by the community at large, then there are two possible avenues of recourse: 51 | 52 | . Have the Lead Maintainer remove the malicious Maintainer 53 | . In the case that the Lead Maintainer themselves is considered to be the rogue agent: fork the project to a new location and continue development there. 54 | 55 | In the case that GitHub itself becomes the rogue entity, there have been numerous discussions about how to move away from GitHub, which have been summarized on the devwiki https://github.com/bitcoin-core/bitcoin-devwiki/wiki/GitHub-alternatives-for-Bitcoin-Core[here^]. 56 | This summary came in part from discussions on https://github.com/bitcoin/bitcoin/issues/20227[this^] GitHub issue. 57 | -------------------------------------------------------------------------------- /controlling-the-wallet.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Wallet locks 2 | :page-nav_order: 80 3 | :page-parent: Wallet 4 | == Controlling the wallet 5 | 6 | As we can see wallet component startup and shutdown is largely driven from outside the wallet codebase from _src/init.cpp_. 7 | 8 | Once the wallet component is started and any wallets supplied via argument have been verified and loaded, wallet functionality ceases to be called from _init.cpp_ and instead is controlled using external programs in a number of ways. 9 | The wallet can be controlled using `bitcoin-cli` or `bitcoin-qt` GUI, and wallet files can be interacted with using the stand-alone `bitcoin-wallet` tool. 10 | 11 | Both `bitcoind` and `bitcoin-qt` run a (JSON) RPC server which is ready to service, amongst other things, commands to interact with wallets. 12 | The command line tool `bitcoin-cli` will allow interaction of any RPC server started by either `bitcoind` or `bitcoin-qt`. 13 | 14 | TIP: If using `bitcoin-qt` there is also an RPC console built into the GUI or you can run with `-server=1` to allow access via `bitcoin-cli`. 15 | 16 | If using the `bitcoin-qt` GUI itself then communication with the wallet is done directly via qt's https://github.com/bitcoin/bitcoin/blob/v23.0/src/qt/walletmodel.h#L51-L52[`WalletModel` interface^]. 17 | 18 | Commands which can be used to control the wallet via RPC are listed in https://github.com/bitcoin/bitcoin/blob/v23.0/src/wallet/rpc/wallet.cpp#L662-L731[_rpcwallet.cpp_^]. 19 | 20 | === Wallet via RPC 21 | 22 | If we take a look at the https://github.com/bitcoin/bitcoin/blob/v23.0/src/wallet/rpc/wallet.cpp#L195-L238[`loadwallet` RPC^] we can see similarities to ``WalletClientImpl``'s `LoadWallets()` function. 23 | 24 | However this time the function will check the `WalletContext` to check that we have a wallet context (in this case a reference to a chain interface) loaded. 25 | Next it will call https://github.com/bitcoin/bitcoin/blob/v23.0/src/wallet/wallet.cpp#L260-L271[`wallet.cpp#LoadWallet`^] which starts by grabbing `g_loading_wallet_mutex` and adding the wallet to `g_loading_wallet_set`, before calling https://github.com/bitcoin/bitcoin/blob/v23.0/src/wallet/wallet.cpp#L227-L257[`LoadWalletInternal`^] which adds the wallet to `vpwallets` and sets up various event notifications. 26 | 27 | Further operation of the wallet RPCs are detailed in their man pages, but one thing to take note of is that whilst `loadwallet()` (and `unloadwallet()`) both take a `wallet_name` argument, the other wallet RPCs do not. 28 | Therefore in order to control a specific wallet from an instance of `bitcoin{d|-qt}` that has multiple wallets loaded, bitcoin-cli must be called with the `-rpcwallet` argument, to specify the wallet which the action should be performed against, e.g. `bitcoin-cli --rpcwallet=your_wallet_name getbalance` 29 | 30 | === Via `bitcoin-cli` tool 31 | 32 | **** 33 | Blockchain Commons contains numerous guides and examples of controlling the wallet using `bitcoin-cli`, including: 34 | 35 | * https://github.com/BlockchainCommons/Learning-Bitcoin-from-the-Command-Line/blob/master/04_0_Sending_Bitcoin_Transactions.md[Sending Bitcoin Transactions^] including using raw transactions 36 | * https://github.com/BlockchainCommons/Learning-Bitcoin-from-the-Command-Line/blob/master/05_0_Controlling_Bitcoin_Transactions.md[Controlling Bitcoin Transactions^] using RBF and CPFP 37 | * https://github.com/BlockchainCommons/Learning-Bitcoin-from-the-Command-Line/blob/master/06_0_Expanding_Bitcoin_Transactions_Multisigs.md[Using multisig^] to send and receive 38 | * https://github.com/BlockchainCommons/Learning-Bitcoin-from-the-Command-Line/blob/master/07_0_Expanding_Bitcoin_Transactions_PSBTs.md[Creating and using PSBTs^] and integrating them with hardware wallets 39 | * https://github.com/BlockchainCommons/Learning-Bitcoin-from-the-Command-Line/blob/master/08_0_Expanding_Bitcoin_Transactions_Other.md[Adding locktimes and OP_RETURN data^] 40 | **** 41 | -------------------------------------------------------------------------------- /css/asciidoc.css: -------------------------------------------------------------------------------- 1 | span.icon > .fa { 2 | cursor: default; 3 | } 4 | .admonitionblock td.icon { 5 | text-align: center; 6 | width: 80px; 7 | } 8 | .admonitionblock td.icon [class^="fa icon-"] { 9 | font-size: 2.5em; 10 | text-shadow: 1px 1px 2px rgba(0,0,0,.5); 11 | cursor: default; 12 | } 13 | .admonitionblock td.icon .icon-note:before { 14 | content: "\f05a"; 15 | color: #19407c; 16 | } 17 | .admonitionblock td.icon .icon-tip:before { 18 | content: "\f0eb"; 19 | text-shadow: 1px 1px 2px rgba(155,155,0,.8); 20 | color: #111; 21 | } 22 | .admonitionblock td.icon .icon-warning:before { 23 | content: "\f071"; 24 | color: #bf6900; 25 | } 26 | .admonitionblock td.icon .icon-caution:before { 27 | content: "\f06d"; 28 | color: #bf3400; 29 | } 30 | .admonitionblock td.icon .icon-important:before { 31 | content: "\f06a"; 32 | color: #bf0000; 33 | } 34 | -------------------------------------------------------------------------------- /cwallet.adoc: -------------------------------------------------------------------------------- 1 | :page-title: CWallet 2 | :page-nav_order: 100 3 | :page-parent: Wallet 4 | include::links-onepage.adoc[] 5 | == CWallet 6 | 7 | The `CWallet` object is the fundamental wallet representation inside Bitcoin Core. 8 | `CWallet` stores transactions and balances and has the ability to create new transactions. 9 | `CWallet` also contains references to the chain interface for the wallet along with storing wallet metadata such as `nWalletVersion`, wallet flags, wallet name and address book. 10 | 11 | === CWallet creation 12 | 13 | The `CWallet` constructor takes a pointer to the chain interface for the wallet, a wallet name and a pointer to the underlying `WalletDatabase`: 14 | 15 | The constructor is not called directly, but instead from the public function `CWallet::Create()`, which is itself called from `CreateWallet()`, `LoadWallets()` (or `TestLoadWallet()`). 16 | In addition to the arguments required by the constructor, `CWallet::Create()` also has a `wallet_flags` argument. 17 | Wallet flags are represented as a single `unit64_t` bit field which encode certain wallet properties: 18 | 19 | .src/wallet/walletutil.h 20 | [source,cpp,options=nowrap] 21 | ---- 22 | enum WalletFlags : uint64_t { 23 | WALLET_FLAG_AVOID_REUSE = (1ULL << 0), 24 | WALLET_FLAG_KEY_ORIGIN_METADATA = (1ULL << 1), 25 | WALLET_FLAG_DISABLE_PRIVATE_KEYS = (1ULL << 32), 26 | WALLET_FLAG_BLANK_WALLET = (1ULL << 33), 27 | WALLET_FLAG_DESCRIPTORS = (1ULL << 34), 28 | WALLET_FLAG_EXTERNAL_SIGNER = (1ULL << 35), 29 | }; 30 | ---- 31 | 32 | See https://github.com/bitcoin/bitcoin/blob/v23.0/src/wallet/walletutil.h#L36-L70[_src/wallet/walletutil.h_^] for additional information on the meanings of the wallet flags. 33 | 34 | `CWallet::Create()` will first attempt to create the `CWallet` object and load it, returning if any errors are encountered. 35 | 36 | If `CWallet::Create` is creating a new wallet -- on its 'first run' -- the wallet version and wallet flags will be set, before either `LegacyScriptPubKeyMan` or ``DescriptorScriptPubKeyMan``'s are setup, depending on whether the `WALLET_FLAG_DESCRIPTORS` flag was set on the wallet. 37 | 38 | Following successful creation, various program arguments are checked and applied to the wallet. 39 | These include options such as `-addresstype`, `-changetype`, `-mintxfee` and `-maxtxfee` amongst others. 40 | It is at this stage that warnings for unusual or unsafe values of these arguments are generated to be returned to the user. 41 | 42 | After the wallet is fully initialized and setup, its keypool will be topped up before the wallet is locked and registered with the Validation interface, which will handle callback notifications generated during the (optional) upcoming chain rescan. 43 | The rescan is smart in detecting the wallet "birthday" using metadata stored in the xref:scriptpubkeymanagers.adoc#scriptpubkeymanagers[SPKM] and won't scan blocks produced before this date. 44 | 45 | Finally, the `walletinterface` is setup for the wallet before the `WalletInstance` is returned to the caller. 46 | -------------------------------------------------------------------------------- /debugging.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Debugging Bitcoin Core 2 | :page-nav_order: 50 3 | :page-parent: Overview and Development Process 4 | ==== Debugging Bitcoin Core 5 | 6 | // Archived @ https://archive.is/hRExH 7 | Fabian Jahr has created a https://github.com/fjahr/debugging_bitcoin[guide^] on "Debugging Bitcoin Core", aimed at detailing the ways in which various Bitcoin Core components can be debugged, including the Bitcoin Core binary itself, unit tests, functional tests along with an introduction to core dumps and the Valgrind memory leak detection suite. 8 | 9 | Of particular note to Developers are the configure flags used to build Bitcoin Core without optimisations to permit more effective debugging of the various resulting binary files. 10 | 11 | Fabian has also presented on this topic a number of times. 12 | A https://btctranscripts.com/scalingbitcoin/tel-aviv-2019/edgedevplusplus/debugging-bitcoin/[transcript^] of his edgedevplusplus talk is available. 13 | -------------------------------------------------------------------------------- /deep-arch.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Deep technical dive 2 | :page-nav_order: 80 3 | :page-parent: Architecture 4 | == Deep technical dive 5 | 6 | lsilva01 has written a deep technical dive into the architecture of Bitcoin Core as part of the Bitcoin Core Onboarding Documentation in https://github.com/chaincodelabs/bitcoin-core-onboarding/blob/main/1.0_bitcoin_core_architecture.asciidoc[Bitcoin Architecture^]. 7 | 8 | Once you've gained some insight into the architecture of the program itself you can learn further details about which code files implement which functionality from the https://github.com/chaincodelabs/bitcoin-core-onboarding/blob/main/1.1_regions.asciidoc[Bitcoin Core regions^] document. 9 | 10 | James O'Beirne has recorded 3 videos which go into detail on how the codebase is laid out, how the build system works, what developer tools there are, as well as what the primary function of many of the files in the codebase are: 11 | 12 | . https://www.youtube.com/watch?v=J1Ru8V36z_Y[Architectural tour of Bitcoin Core (part 1 of 3)^] 13 | . https://www.youtube.com/watch?v=RVWcUnpZX4E[Architectural tour of Bitcoin Core (part 2 of 3)^] 14 | . https://www.youtube.com/watch?v=UiD5DZU9Zp4[Architectural tour of Bitcoin Core (part 3 of 3)^] 15 | 16 | ryanofsky has written a handy https://github.com/ryanofsky/bitcoin/blob/pr/libs/doc/design/libraries.md[guide^] covering the different libraries contained within Bitcoin Core, along with some of their conventions and a dependency graph for them. 17 | Generally speaking, the desire is for the Bitcoin Core project to become more modular and less monolithic over time. 18 | -------------------------------------------------------------------------------- /design-principles.adoc: -------------------------------------------------------------------------------- 1 | :page-title: General design principles 2 | :page-nav_order: 10 3 | :page-parent: Architecture 4 | == General design principles 5 | 6 | Over the last decade, as the scope, complexity and test coverage of the codebase has increased, there has been a general effort to not only break Bitcoin Core down from its monolithic structure but also to move towards it being a collection of self-contained subsystems. 7 | The rationale for such a goal is that this makes components easier to reason about, easier to test, and less-prone to layer violations, as subsystems can contain a full view of all the information they need to operate. 8 | 9 | Subsystems can be notified of events relevant to them and take appropriate actions on their own. 10 | On the GUI/QT side this is handled with signals and slots, but in the core daemon this is largely still a producer/consumer pattern. 11 | 12 | The various subsystems are often suffixed with `Manager` or `man`, e.g. `CConnman` or `ChainstateManager`. 13 | 14 | TIP: The extra "C" in `CConnman` is a hangover from the https://en.wikipedia.org/wiki/Hungarian_notation[Hungarian notation^] used originally by Satoshi. 15 | This is being phased out as-and-when affected code is touched during other changes. 16 | 17 | You can see some (but not all) of these subsystems being initialized in https://github.com/bitcoin/bitcoin/blob/v23.0/src/init.cpp#L1113[_init.cpp#AppInitMain()_^]. 18 | 19 | There is a recent preference to favour python over bash/sh for scripting, e.g. for linters, but many shell scripts remain in place for CI and contrib/ scripts. 20 | -------------------------------------------------------------------------------- /docinfo-footer.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 34 | -------------------------------------------------------------------------------- /docinfo.html: -------------------------------------------------------------------------------- 1 | 7 | 8 | 12 | 13 | 28 | 29 | 35 | 36 | 41 | -------------------------------------------------------------------------------- /entries/bugs.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Quickstart: fixing bugs" 4 | nav_exclude: true 5 | --- 6 | # Quickstart: fixing bugs 7 | 8 | {% include best-way.md %} 9 | 10 | In this guide, we assume that you think you've discovered a bug. You 11 | may want to immediately report the bug, or you may want to begin 12 | debugging and fixing it yourself. 13 | 14 | - To report an issue: 15 | 16 | - You may want to quickly [mention](/comms.html) the behavior you 17 | think is a bug in chat to confirm it's unexpected behavior. 18 | 19 | - You should get [set up with GitHub](/github.html) to submit an 20 | issue. 21 | 22 | - To debug an issue: 23 | 24 | - You will need to [build Bitcoin 25 | Core](/building.html) optionally with [debugging 26 | options](/debugging.html). 27 | 28 | - You may want to [write a test](/testing.html) that demonstrates your 29 | bug and will prove it's been fixed. 30 | 31 | - If you need certain network activity to reproduce your bug, you can 32 | test on [test networks](/testnets.html). 33 | 34 | - If you're trying to find a particular file, learn about Bitcoin 35 | Core's [architecture](/architecture.html). 36 | 37 | - If you begin to suspect that your "bug" might be deliberate 38 | behavior, research the [history of particular 39 | code](/past-changes.html). 40 | 41 | - To contribute a change: 42 | 43 | - Follow project [guidelines](/contributing.html) for effective 44 | commits and GitHub PRs. 45 | 46 | - Understand the jargon and process other developers will use in 47 | [evaluating](/pr-maturation.html) your proposed change. 48 | 49 | - Optionally help [review](/reviewing.html) contributions from other 50 | developers to encourage them to review your contribution. 51 | 52 | Many other parts of the contributing guide can help explain how to 53 | successfully debug, report, and fix a bug. If you find yourself stymied 54 | at any point, or just curious about Bitcoin Core, please consider 55 | reading the full documentation starting from the [main page](/). 56 | -------------------------------------------------------------------------------- /entries/first-contribution.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Quickstart: first contribution" 4 | nav_exclude: true 5 | --- 6 | # Quickstart: first contribution 7 | 8 | {% include best-way.md %} 9 | 10 | This guide is for developers who want to quickly begin contributing 11 | to Bitcoin Core but don't know what to work on or how to get started. 12 | 13 | - Find an [entry point](/starting-ideas.html) such as a bug an 14 | established contributor has tagged as a "good first issue". 15 | 16 | - You may want to quickly [mention](/comms.html) your planned approach 17 | in chat to confirm established contributors agree that it's a good 18 | direction. 19 | 20 | - Note on the issue, PR, or other page that you're working on the 21 | problem so others don't duplicate your work. 22 | 23 | - To address an issue: 24 | 25 | - You will need to [build Bitcoin 26 | Core](/building.html) optionally with [debugging 27 | options](/debugging.html). 28 | 29 | - You may want to [write a test](/testing.html) that demonstrates your 30 | bug and will prove it's been fixed. 31 | 32 | - If you need certain network activity to reproduce your bug, you can 33 | test on [test networks](/testnets.html). 34 | 35 | - If you're trying to find a particular file, learn about Bitcoin 36 | Core's [architecture](/architecture.html). 37 | 38 | - If you encounter confusing code, research the [history of that 39 | particular code](/past-changes.html). 40 | 41 | - To contribute a change: 42 | 43 | - Follow project [guidelines](/contributing.html) for effective 44 | commits and GitHub PRs. 45 | 46 | - Understand the jargon and process other developers will use in 47 | [evaluating](/pr-maturation.html) your proposed change. 48 | 49 | - Optionally help [review](/reviewing.html) contributions from other 50 | developers to encourage them to review your contribution. 51 | 52 | Many other parts of the contributing guide can help explain how to 53 | successfully make your first contribution. If you find yourself stymied 54 | at any point, or just curious about Bitcoin Core, please consider 55 | reading the full documentation starting from the [main page](/). 56 | -------------------------------------------------------------------------------- /exercise-intro.adoc: -------------------------------------------------------------------------------- 1 | include::links-onepage.adoc[] 2 | 3 | **** 4 | Using either `bitcoin-cli` in your terminal, or a xref:contributor-exercises.adoc#test_shell_nodes[Jupyter notebook] in conjunction with the `TestShell` class from the Bitcoin Core Test Framework, try to complete the following exercises. 5 | 6 | Changes to the codebase will require you to re-compile afterwards. 7 | 8 | Don't forget to use the compiled binaries found in your source directory, for example `/home/user/bitcoin/src/bitcoind`, otherwise your system might select a previously-installed (non-modified) version of bitcoind. 9 | **** 10 | -------------------------------------------------------------------------------- /exercise_base.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 3, 6 | "metadata": { 7 | "pycharm": { 8 | "name": "#%%\n" 9 | } 10 | }, 11 | "outputs": [], 12 | "source": [ 13 | "import sys, os\n", 14 | "from test_framework.test_shell import TestShell\n", 15 | "\n", 16 | "sys.path.insert(0, os.path.expanduser(\"~/bitcoin/test/functional\"))\n", 17 | "test = TestShell().setup(\n", 18 | " num_nodes=2,\n", 19 | " setup_clean_chain=True,\n", 20 | " extra_args=[[], ['-fallbackfee=0.0002']],\n", 21 | ")" 22 | ] 23 | } 24 | ], 25 | "metadata": { 26 | "kernelspec": { 27 | "display_name": "Python 3 (ipykernel)", 28 | "language": "python", 29 | "name": "python3" 30 | }, 31 | "language_info": { 32 | "codemirror_mode": { 33 | "name": "ipython", 34 | "version": 3 35 | }, 36 | "file_extension": ".py", 37 | "mimetype": "text/x-python", 38 | "name": "python", 39 | "nbconvert_exporter": "python", 40 | "pygments_lexer": "ipython3", 41 | "version": "3.9.7" 42 | } 43 | }, 44 | "nbformat": 4, 45 | "nbformat_minor": 1 46 | } -------------------------------------------------------------------------------- /fork-wishlist.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Fork wish lists 2 | :page-nav_order: 50 3 | :page-parent: Consensus and Validation 4 | == Fork wish lists 5 | 6 | There are a number of items that developers have had on their wish lists to tidy up in future fork events. 7 | 8 | An https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2019-March/016714.html[email^] from Matt Corallo with the subject "The Great Consensus Cleanup" described a "wish list" of items developers were keen to tidy up in a future soft fork. 9 | 10 | The Hard Fork Wishlist is described on this en.bitcoin.it/wiki https://en.bitcoin.it/wiki/Hardfork_Wishlist[page^]. 11 | The rationale for collecting these changes together, is that if backwards-incompatible (hard forking) changes are being made, then we "might as well" try and get a few in at once, as these events are so rare. 12 | -------------------------------------------------------------------------------- /github.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Development workflow 2 | :page-nav_order: 10 3 | :page-parent: Overview and Development Process 4 | == Development workflow 5 | 6 | Bitcoin Core uses a GitHub-based workflow for development. 7 | The primary function of GitHub in the workflow is to discuss patches and connect them with review comments. 8 | 9 | While some other prominent projects, e.g. the Linux kernel, use email to solicit feedback and review, Bitcoin Core has used GitHub for many years. 10 | Initially, Satoshi distributed the code through private emails and hosting source archives at bitcoin.org, and later by hosting on SourceForge (which used SVN but did not at that time have a pull request system like GitHub). 11 | The earliest reviewers submitted changes using patches either through email exchange with Satoshi, or by posting them on the bitcoin forum. 12 | 13 | In August 2009, the source code was moved to GitHub by Sirius, and development has remained there and used the GitHub workflows ever since. 14 | 15 | === Use of GitHub 16 | 17 | The GitHub side of the Bitcoin Core workflow for Contributors consists primarily of: 18 | 19 | * Issues 20 | * PRs 21 | * Reviews 22 | * Comments 23 | 24 | Generally, issues are used for two purposes: 25 | 26 | . Posting known issues with the software, e.g., bug reports, crash logs 27 | . Soliciting feedback on potential changes without providing associated code, as would be required in a PR. 28 | 29 | GitHub provides their own https://guides.github.com/features/issues/[guide^] on mastering Issues which is worth reading to understand the feature-set available when working with an issue. 30 | 31 | PRs are where Contributors can submit their code against the main codebase and solicit feedback on the concept, the approach taken for the implementation, and the actual implementation itself. 32 | 33 | PRs and Issues are often linked to/from one another: 34 | 35 | [example] 36 | ==== 37 | One common workflow is when an Issue is opened to report a bug. 38 | After replicating the issue, a Contributor creates a patch and then opens a PR with their proposed changes. 39 | 40 | In this case, the Contributor should, in addition to comments about the patch, reference that the patch fixes the issue. 41 | For a patch which fixes issue 22889 this would be done by writing "fixes #22889" in the PR description or in a commit message. 42 | In this case, the syntax "fixes #issue-number" is caught by GitHub's https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue[pull request linker^], which handles the cross-link automatically. 43 | ==== 44 | 45 | Another use-case of Issues is soliciting feedback on ideas that might require _significant_ changes. 46 | This helps free the project from having too many PRs open which aren't ready for review and might waste reviewers' time. 47 | In addition, this workflow can also save Contributors their _own_ valuable time, as an idea might be identified as unlikely to be accepted _before_ the contributor spends their time writing the code for it. 48 | 49 | Most code changes to bitcoin are proposed directly as PRs -- there's no need to open an Issue for every idea before implementing it unless it may require significant changes. 50 | Additionally, other Contributors (and would-be Reviewers) will often agree with the approach of a change, but want to "see the implementation" before they can really pass judgement on it. 51 | 52 | GitHub is therefore used to help store and track reviews to PRs in a public way. 53 | 54 | Comments (inside Issues, PRs, Projects etc.) are where all (GitHub) users can discuss relevant aspects of the item and have history of those discussions preserved for future reference. 55 | Often Contributors having "informal" discussions about changes on e.g. IRC will be advised that they should echo the gist of their conversation as a comment on GitHub, so that the rationale behind changes can be more easily determined in the future. 56 | -------------------------------------------------------------------------------- /gui-building.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Building the GUI 2 | :page-nav_order: 10 3 | :page-parent: GUI 4 | include::links-onepage.adoc[] 5 | == Building the GUI 6 | 7 | `bitcoin-qt`, which includes the QT GUI with the node, is built automatically when the build dependencies are met. 8 | Required packages to meet dependencies can be found in the build instructions in _src/doc/build-*.md_ as appropriate for your platform. 9 | If you have the required packages installed but do not wish to build the `bitcoin-qt` then you must run `./configure` with the option `--with-gui=no`. 10 | 11 | [NOTE] 12 | ==== 13 | If the build is configured with `--enable-multiprocess` then additional binaries will be built: 14 | 15 | . `bitcoin-node` 16 | . `bitcoin-wallet` 17 | . `bitcoin-gui` 18 | ==== 19 | 20 | == Qt 21 | 22 | QT is currently very intertwined with the rest of the codebase. 23 | See the library xref:library-structure.adoc#library-dependency-graph[depencency graph] for more context. 24 | 25 | Developers would ideally like to reduce these dependencies in the future. 26 | 27 | == Qt documentation 28 | 29 | There is useful documentation for developers looking to contribute to the Qt side of the codebase found at https://github.com/bitcoin-core/bitcoin-devwiki/wiki//Developer-Notes-for-Qt-Code[Developer Notes for Qt Code^]. 30 | 31 | -------------------------------------------------------------------------------- /gui-initialization.adoc: -------------------------------------------------------------------------------- 1 | :page-title: GUI initialization 2 | :page-nav_order: 20 3 | :page-parent: GUI 4 | include::links-onepage.adoc[] 5 | == Main GUI program 6 | 7 | The loading point for the GUI is _src/qt/main.cpp_. 8 | `main()` calls `GuiMain()` from _src/qt/bitcoin.cpp_, passing along any program arguments with it. 9 | `GuiMain` starts by calling `SetupEnvironment()` which amongst other things, configures the runtime locale and charset. 10 | 11 | Next an empty `NodeContext` is set up, which is then populated into a fully-fledged node interface via being passed to `interfaces::MakeNode()`, which returns an `interfaces::Node`. 12 | Recall that in xref:wallet-init.adoc#wallet_component_initialisation[wallet component initialization] we also saw the wallet utilizing a `NodeContext` as part of its `WalletInitInterface`. 13 | In both cases the `NodeContext` is being used to pass chain and network references around without needing to create globals. 14 | 15 | After some QT setup, command-line and application arguments are parsed. 16 | What follows can be outlined from the code comments: 17 | 18 | [start=3] 19 | . Application identification 20 | . Initialization of translations, so that intro dialogue is in user's language 21 | . Now that settings and translations are available, ask user for data directory 22 | . Determine availability of data directory and parse bitcoin.conf 23 | . Determine network (and switch to network specific options) 24 | . URI IPC sending 25 | . Main GUI initialization 26 | 27 | == GUI initialisation 28 | 29 | After configuration the GUI is initialized. 30 | Here the `Node` object created earlier is passed to `app.SetNode()` before a window is created and the application executed. 31 | 32 | The bulk of the Qt GUI classes are defined in _src/qt/bitcoingui.{h|cpp}_. 33 | 34 | -------------------------------------------------------------------------------- /gui-motivation.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Motivation for a GUI 2 | :page-nav_order: 0 3 | :page-parent: GUI 4 | == Motivation for a GUI 5 | 6 | Bitcoin Core has shipped with a GUI since the first version. 7 | Originally this was a wxWidgets GUI, but in 2011 a move to QT was https://github.com/bitcoin/bitcoin/pull/521[completed]. 8 | Satoshi originally had plans to have a decentralized market place and even poker game inside Bitcoin, so including a GUI, which also had wallet and address book functionality, made sense from the get-go. 9 | 10 | The motivation to _continue_ to include a GUI with Bitcoin Core today is for accessibility. 11 | New users can access a best-in-class Bitcoin experience via a single software package. 12 | It's not safe or realistic to expect users to download multiple programs and connect them securely into a software suite, just to use bitcoin. 13 | 14 | It does not have to be the prettiest UI, but needs to provide the functionality to use bitcoin. 15 | It is possible to connect other frontends to Bitcoin Core, but they are connected via RPCs, and do not have the first-class interface (to the node component) that the bundled GUI has. 16 | -------------------------------------------------------------------------------- /gui.adoc: -------------------------------------------------------------------------------- 1 | = GUI 2 | :page-nav_order: 40 3 | :page-has_children: true 4 | 5 | TIP: This section has been updated to Bitcoin Core @ https://github.com/bitcoin/bitcoin/tree/v23.0[v23.0^] 6 | 7 | The GUI has its own separate repo at https://github.com/bitcoin-core/gui[bitcoin-core/gui^]. 8 | PRs which primarily target the GUI should be made here, and then they will get merged into the primary repo. 9 | Developer Marco Falke created https://github.com/MarcoFalke/bitcoin-core/issues/26[an issue^] in his fork which detailed some of the rationale for the split, but essentially it came down to: 10 | 11 | . Separate issue and patch management 12 | . More focused review and interests 13 | . Maintain high quality assurance 14 | 15 | He also stated that: 16 | 17 | [quote, Marco Falke] 18 | ____ 19 | Splitting up the GUI (and splitting out modules in general) has been brought up often in recent years. Now that the GUI is primarily connected through interfaces with a bitcoin node, it seems an appropriate time to revive this discussion. 20 | ____ 21 | 22 | https://github.com/bitcoin/bitcoin/pull/19071[PR#19071^] contained the documentation change now contained in the Bitcoin Core primary repository, along with details of the monotree approach that was ultimately taken. 23 | The documentation change provides guidance on what a "GUI change" is: 24 | 25 | [quote, src/CONTRIBUTING.md] 26 | ____ 27 | As a rule of thumb, everything that only modifies `src/qt` is a GUI-only pull 28 | request. However: 29 | 30 | * For global refactoring or other transversal changes the node repository 31 | should be used. 32 | * For GUI-related build system changes, the node repository should be used 33 | because the change needs review by the build systems reviewers. 34 | * Changes in `src/interfaces` need to go to the node repository because they 35 | might affect other components like the wallet. 36 | 37 | For large GUI changes that include build system and interface changes, it is 38 | recommended to first open a PR against the GUI repository. When there 39 | is agreement to proceed with the changes, a PR with the build system 40 | and interfaces changes can be submitted to the node repository. 41 | ____ 42 | 43 | On a related note, another https://github.com/bitcoin/bitcoin/issues/24045[issue^] was recently opened by Falke, to discuss the possibility of instituting the same monotree changes for wallet code. 44 | 45 | -------------------------------------------------------------------------------- /hardcoded-consensus-values.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Hardcoded consensus values 2 | :page-nav_order: 80 3 | :page-parent: Consensus and Validation 4 | == Hardcoded consensus values 5 | 6 | _consensus/consensus.h_ contains a number of `static const` values relating to consensus rules. 7 | These are globally shared between files such as _validation.cpp_, _rpc_mining.cpp_ and _rpc/mining.cpp_. 8 | These consensus-critical values are marked as `const` so that there is no possibility that they can be changed at any point during program execution. 9 | 10 | One example of this would be the maximum block weight which should not ever be exceeded: 11 | 12 | [source,cpp] 13 | ---- 14 | static const unsigned int MAX_BLOCK_WEIGHT = 4000000; 15 | ---- 16 | 17 | _consensus/amount.h_ contains the conversion rate between satoshis and one "bitcoin", as well as a `MAX_MONEY` constant. 18 | These are marked as `constexpr` to indicate that they should be evaluated at compile time and then remain as `const` during execution. 19 | 20 | [source,cpp] 21 | ---- 22 | /** The amount of satoshis in one BTC. */ 23 | static constexpr CAmount COIN = 100000000; 24 | 25 | /** No amount larger than this (in satoshi) is valid. 26 | * 27 | * Note that this constant is *not* the total money supply, which in Bitcoin 28 | * currently happens to be less than 21,000,000 BTC for various reasons, but 29 | * rather a sanity check. As this sanity check is used by consensus-critical 30 | * validation code, the exact value of the MAX_MONEY constant is consensus 31 | * critical; in unusual circumstances like a(nother) overflow bug that allowed 32 | * for the creation of coins out of thin air modification could lead to a fork. 33 | * */ 34 | static constexpr CAmount MAX_MONEY = 21000000 * COIN; 35 | ---- 36 | 37 | [TIP] 38 | ==== 39 | Do you think that the `COIN` constant is necessary at a consensus level, or is it a Bitcoin Core-specific abstraction? 40 | ==== 41 | -------------------------------------------------------------------------------- /images/compilation_firewall_amiti.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chaincodelabs/onboarding-to-bitcoin-core/63309556fb4fd8b64bed5e2e5ffd6173dedda826/images/compilation_firewall_amiti.jpg -------------------------------------------------------------------------------- /images/jupyter_duplicate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chaincodelabs/onboarding-to-bitcoin-core/63309556fb4fd8b64bed5e2e5ffd6173dedda826/images/jupyter_duplicate.png -------------------------------------------------------------------------------- /images/pimpl_peerman_amiti.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chaincodelabs/onboarding-to-bitcoin-core/63309556fb4fd8b64bed5e2e5ffd6173dedda826/images/pimpl_peerman_amiti.png -------------------------------------------------------------------------------- /images/pimpl_txrequest_amiti.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chaincodelabs/onboarding-to-bitcoin-core/63309556fb4fd8b64bed5e2e5ffd6173dedda826/images/pimpl_txrequest_amiti.png -------------------------------------------------------------------------------- /index.adoc: -------------------------------------------------------------------------------- 1 | = Onboarding to Bitcoin Core 2 | include::settings.adoc[] 3 | :leveloffset: +1 4 | 5 | include::all_chapters.adoc[] 6 | 7 | :leveloffset: -1 8 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | --- 2 | # Feel free to add content and custom Front Matter to this file. 3 | # To modify the layout, see https://jekyllrb.com/docs/themes/#overriding-theme-defaults 4 | 5 | layout: home 6 | --- 7 | 31 | 32 |
33 |
34 |

Get onboarded

35 |

Read the entire onboarding guide in a 36 | convenient single-page format.

37 |

Alternatively, view as a PDF or 38 | ePub.

39 |
40 | 41 |
42 |

Review a PR

43 |

Prevent problems before they affect your bitcoins. Choose a PR to 44 | review and learn as you go.

45 |
46 | 47 |
48 |

Make a contribution

49 |

Get started making your first contribution.

50 |
51 | 52 |
53 |

Squash a bug

54 |

The minimal information you need to fix a problem.

55 |
56 | 57 |
58 | 59 |
60 | Document originally written by Will Clark. 61 |
62 | -------------------------------------------------------------------------------- /index_epub.adoc: -------------------------------------------------------------------------------- 1 | = Onboarding to Bitcoin Core 2 | include::settings.adoc[] 3 | :doctype: book 4 | :lang: en 5 | :revdate: 2023-12-04 6 | :doctitle: Onboarding to Bitcoin Core 7 | :author: Will Clark 8 | :description: Onboarding documentation for Bitcoin Core developers. 9 | :keywords: bitcoin, core, crypto, p2p, consensus, node, wallet, docs 10 | :copyright: MIT Licensed 11 | :source: https://github.com/chaincodelabs/onboarding-to-bitcoin-core/ 12 | :mermaid-format: png 13 | :mermaid-scale: 2 14 | 15 | :leveloffset: +1 16 | 17 | include::all_chapters.adoc[] 18 | 19 | :leveloffset: -1 20 | -------------------------------------------------------------------------------- /index_pdf.adoc: -------------------------------------------------------------------------------- 1 | = Onboarding to Bitcoin Core 2 | include::settings.adoc[] 3 | :leveloffset: +1 4 | :mermaid-format: png 5 | :mermaid-scale: 2 6 | 7 | include::all_chapters.adoc[] 8 | 9 | :leveloffset: -1 10 | -------------------------------------------------------------------------------- /jekyll_workflow.md: -------------------------------------------------------------------------------- 1 | # Jekyll workflow and deployment 2 | 3 | Although the documentation is written in asciidoc, it can also be 4 | deployed as a website using the the Jekyll static site generator. 5 | The Jekyll version creates one page for each group of related subjects, 6 | over 100 pages in total. 7 | 8 | Operation of Jekyll has been wrapped in a Makefile for convenience: 9 | 10 | - `make build` will build the site and all assets into the `_site` 11 | directory for hosting 12 | 13 | - `make preview` will build the site and run a preview on a lightweight 14 | HTTP server on the port printed on the command line 15 | 16 | - `make all` will build the site and run a number of tests. It is 17 | recommended to do this before publishing 18 | 19 | - `make clean` will clean up files generated by the build process 20 | 21 | Jekyll and the build process both use asciidoctor, so you need all the 22 | same depedencies for adoc building plus the Jekyll depedencies and GNU 23 | make. To simplify replication of this process a `Containerfile` is 24 | provided in the repository. It should be compatible with the `docker` 25 | command although it was developed with the entirely free software 26 | `podman` command. An example use: 27 | 28 | ``` 29 | REPO=~/repos/onboarding-to-bitcoin-core/ 30 | podman build -v $REPO:/srv/ -t guide $REPO 31 | id=$( podman run --cap-add NET_ADMIN --cap-add NET_RAW --detach -v $REPO:/srv/ -p 0.0.0.0:8080:4000 -it localhost/guide:latest ) 32 | podman attach $id 33 | ``` 34 | 35 | Or for Docker: 36 | 37 | ``` 38 | REPO=~/repos/onboarding-to-bitcoin-core/ 39 | 40 | # docker build 41 | docker build -f $REPO/Dockerfile -t guide $REPO 42 | # or with docker buildx 43 | docker build --load -f $REPO/Dockerfile -t guide $REPO 44 | 45 | id=$( docker run --detach -v $REPO:/srv/ -p 8080:4000 -it guide:latest ) 46 | docker attach $id 47 | 48 | bundle install # shouldn't be needed after Dockerfile build, but it is... 49 | ``` 50 | -------------------------------------------------------------------------------- /library-structure.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Library structure 2 | :page-nav_order: 50 3 | :page-parent: Architecture 4 | :mermaid-puppeteer-config: ./puppeteer-config.json 5 | == Library structure 6 | 7 | Bitcoin Core compilation outputs a number of libraries, some which are designed to be used internally, and some which are designed to be re-used by external applications. 8 | The internally-used libraries generally have unstable APIs making them unsuitable for re-use, but `libbitcoin_consensus` and `libbitcoin_kernel` are designed to be re-used by external applications. 9 | 10 | Bitcoin Core has a https://github.com/bitcoin/bitcoin/blob/master/doc/design/libraries.md[guide^] which describes the various libraries, their conventions, and their various dependencies. 11 | The dependency graph is shown below for convenience, but may not be up-to-date with the Bitcoin Core document. 12 | 13 | [id=library-dependency-graph] 14 | .Bitcoin library dependency graph 15 | [mermaid, target=bitcoin-lib-dependencies] 16 | .... 17 | flowchart TB 18 | bitcoin-wallet --> libbitcoin_wallet_tool 19 | bitcoin-wallet --> libbitcoin_wallet 20 | 21 | bitcoin-qt ---> libbitcoin_wallet 22 | bitcoin-qt ---> libbitcoinqt 23 | bitcoin-qt ---> libbitcoin_node 24 | 25 | bitcoind ---> libbitcoin_wallet 26 | bitcoind --> libbitcoin_node 27 | 28 | bitcoin-cli ---> libbitcoin-cli 29 | 30 | libbitcoin_wallet_tool --> libbitcoin_wallet 31 | libbitcoin_wallet_tool --> libbitcoin_util 32 | 33 | libbitcoin-cli --> libbitcoin_common 34 | libbitcoin-cli --> libbitcoin_util 35 | 36 | libbitcoin_node --> libbitcoin_common 37 | libbitcoin_node --> libbitcoin_consensus 38 | libbitcoin_node --> libbitcoin_kernel 39 | libbitcoin_node --> libbitcoin_util 40 | 41 | libbitcoinqt --> libbitcoin_util 42 | libbitcoinqt --> libbitcoin_common 43 | 44 | libbitcoin_wallet --> libbitcoin_util 45 | libbitcoin_wallet --> libbitcoin_common 46 | 47 | libbitcoin_common --> libbitcoin_util 48 | libbitcoin_common --> libbitcoin_consensus 49 | 50 | libbitcoin_kernel --> libbitcoin_consensus 51 | libbitcoin_kernel --> libbitcoin_util 52 | 53 | classDef types fill:green,color:white,stroke:green; 54 | class bitcoin-wallet,bitcoind,bitcoin-cli,bitcoin-qt types 55 | .... 56 | 57 | It follows that API changes to the libraries which are internally-facing can be done slightly easier than for libraries with externally-facing APIs, for which more care for compatibility must be taken. 58 | -------------------------------------------------------------------------------- /links-onepage.adoc: -------------------------------------------------------------------------------- 1 | :single-transactions: 2 | :appendix: 3 | :multiple-transactions: 4 | :transactions-from-blocks: 5 | :validating-scripts: 6 | :address-relay: 7 | :block-relay: 8 | :bootstrapping: 9 | :p2p-violations: 10 | :message-relay: 11 | :input-checks: 12 | :transaction-relay: 13 | :mempool-tx-format: 14 | :consensus-bugs: 15 | :locating-consensus-code: 16 | :transaction-validation: 17 | :wallet-init: 18 | :library-structure: 19 | :scriptpubkeymanagers: 20 | :bips: 21 | :subtrees: 22 | :consensus-v-policy: 23 | :constructing-transactions: 24 | :transaction-identification: 25 | :p2p-attacks: 26 | :threads: 27 | :mempool-lifecycle: 28 | :contributor-exercises: 29 | :wallet-database: 30 | :script-appendix: 31 | -------------------------------------------------------------------------------- /links-separate.adoc: -------------------------------------------------------------------------------- 1 | :single-transactions: single-transactions.adoc# 2 | :appendix: appendix.adoc# 3 | :multiple-transactions: multiple-transactions.adoc# 4 | :transactions-from-blocks: transactions-from-blocks.adoc# 5 | :validating-scripts: validating-scripts.adoc# 6 | :address-relay: address-relay.adoc# 7 | :block-relay: block-relay.adoc# 8 | :address-relay: address-relay.adoc# 9 | :transaction-relay: transaction-relay.adoc# 10 | :bootstrapping: bootstrapping.adoc# 11 | :p2p-violations: p2p-violations.adoc# 12 | :message-relay: message-relay.adoc# 13 | :input-checks: input-checks.adoc# 14 | :mempool-tx-format: mempool-tx-format.adoc# 15 | :consensus-bugs: consensus-bugs.adoc# 16 | :locating-consensus-code: locating-consensus-code.adoc# 17 | :transaction-validation: transaction-validation.adoc# 18 | :wallet-init: wallet-init.adoc# 19 | :library-structure: library-structure.adoc# 20 | :scriptpubkeymanagers: scriptpubkeymanagers.adoc# 21 | :bips: bips.adoc# 22 | :subtrees: subtrees.adoc# 23 | :consensus-v-policy: consensus-v-policy.adoc# 24 | :constructing-transactions: constructing-transactions.adoc# 25 | :transaction-identification: transaction-identification.adoc# 26 | :p2p-attacks: p2p-attacks# 27 | :threads: threads.adoc# 28 | :mempool-lifecycle: mempool-lifecycle.adoc# 29 | :contributor-exercises: contributor-exercises.adoc# 30 | :wallet-database: wallet-database.adoc# 31 | :script-appendix: script-appendix.adoc# 32 | -------------------------------------------------------------------------------- /locating-consensus-code.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Locating consensus code 2 | :page-nav_order: 0 3 | :page-parent: Consensus and Validation 4 | include::links-onepage.adoc[] 5 | [[consensus_in_bitcoin_core]] 6 | == Consensus in Bitcoin Core 7 | 8 | Naturally one might assume that all code related to consensus could be found in the _src/consensus/_ directory, however this is not entirely the case. 9 | Components of consensus-related code can be found across the Bitcoin Core codebase in a number of files, including but not limited to: 10 | 11 | [#consensus-components,listing] 12 | ---- 13 | 📂 bitcoin 14 | 📂 src 15 | 📂 consensus 16 | 📂 script 17 | 📄interpreter.cpp 18 | 📄 validation.h 19 | 📄 validation.cpp 20 | ---- 21 | 22 | Consensus-critical functions can also be found in proximity to code which could affect whether a node considers a transaction or block valid. 23 | This could extend to, for example, block storage xref:consensus-bugs.adoc#database_consensus[database] code. 24 | 25 | An abbreviated list of some of the more notable consensus functions and variables is shown below. 26 | 27 | .Some consensus functions and variables 28 | [cols="2,4"] 29 | |=== 30 | |File |Objects 31 | 32 | |_src/consensus/amount.h_ 33 | |`COIN`, `MAX_MONEY`, `MoneyRange()` 34 | 35 | |_src/consensus/consensus.h_ 36 | |`BLOCK{SIZE\|WEIGHT\|SIGOPS_COST}`, `COINBASE_MATURITY`, `WITNESS_SCALE_FACTOR`, `MIN_TX_WEIGHT` 37 | 38 | |_src/consensus/merkle.{h\|cpp}_ 39 | |`ComputeMerkleRoot(),` `BlockMerkleRoot(),` `BlockWitnessMerkleRoot()` 40 | 41 | |_src/consensus/params.h_ 42 | |`BuriedDeployment`, `Params`(buried blocks which are valid but known to fail default script verify checks, BIP height activations, PoW params) 43 | 44 | |_src/consensus/tx_check.{h\|cpp}_ 45 | |`CheckTransaction()` 46 | 47 | |_src/consensus/tx_verify.{h\|cpp}_ 48 | |`CheckTxInputs(),` `Get{Legacy}SigOpCount()`, `IsFinalTx(),` `SequenceLock(s)()` 49 | 50 | |_src/consensus/validation.h_ 51 | |`TxValidationResult` (validation result reason), `BlockValidationResult` (validation result reason), `ValidationState`, `Get{Transaction\|Block\|TransactionInput}Weight()` 52 | 53 | |=== 54 | -------------------------------------------------------------------------------- /manfunc.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Manual testing of a functional test 2 | :page-nav_order: 100 3 | :page-parent: Overview and Development Process 4 | ==== Manual testing while running a functional test 5 | 6 | Running regtest as described allows you to start from scratch with an empty chain, empty wallet, and no existing state. 7 | 8 | An effective way to use regtest is to start a https://github.com/chaincodelabs/bitcoin-core-onboarding/blob/main/functional_test_framework.asciidoc[functional test^] and insert a python debug breakpoint. 9 | You can set a breakpoint in a test by adding `import pdb; pdb.set_trace()` at the desired stopping point; when the script reaches this point you'll see the debugger's `(Pdb)` prompt, at which you can type `help` and see and do all kinds of useful things. 10 | 11 | While the (Python) test is paused, you can still control the node using `bitcoin-cli`. 12 | First you need to look up the data directory for the node(s), as below: 13 | 14 | [source,bash,options=nowrap] 15 | ---- 16 | $ ps alx | grep bitcoind 17 | 0 1000 57478 57476 20 0 1031376 58604 pipe_r SLl+ pts/10 0:06 /g/bitcoin/src/bitcoind -datadir=/tmp/bitcoin_func_test_ovsi15f9/node0 -logtimemicros -debug (...) 18 | 0 1000 57479 57476 20 0 965964 58448 pipe_r SLl+ pts/10 0:06 /g/bitcoin/src/bitcoind -datadir=/tmp/bitcoin_func_test_ovsi15f9/node1 -logtimemicros -debug (...) 19 | ---- 20 | 21 | With the `-datadir` path you can look at the `bitcoin.conf` files within the data directories to see what config options are being specified for the test (there's always `regtest=1`) in addition to the runtime options, which is a good way to learn about some advanced uses of regtest. 22 | 23 | In addition to this, we can use the `-datadir=` option with `bitcoin-cli` to control specific nodes, e.g.: 24 | 25 | [source,bash,options=nowrap] 26 | ---- 27 | $ src/bitcoin-cli -datadir=/tmp/bitcoin_func_test_ovsi15f9/node0 getblockchaininfo 28 | ---- 29 | 30 | //// 31 | Add `import time; time.sleep(600)` somewhere into a functional test (which is just a Python script) to suspend the test at that point for 10 minutes. (you may want to add a `print("paused")` statement just before the call to `sleep()` to know it's been reached) 32 | 33 | This has the further advantage (over calling `time.sleep()`), in that you can single-step through the test while also manually interacting with the nodes, combining automated and manual testing. 34 | 35 | //// 36 | -------------------------------------------------------------------------------- /mempool-purpose.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Mempool purpose 2 | :page-nav_order: 10 3 | :page-parent: Mempool 4 | == Mempool purpose 5 | 6 | . The mempool is designed to hold a list of unconfirmed-but-valid transactions that the node has learned about. 7 | . Miners will select transactions from the mempool for assembly into a block using the `getblocktemplate` RPC. 8 | . Transactions have to pass all policy and validation checks before being allowed to enter the mempool. + 9 | The mempool therefore also acts as DoS protection for the node. 10 | . Transactions will not be added to the mempool if they do not meet fee requirements, are non-standard, or double-spend an input of a transaction already in the mempool (excluding BIP 125 RBF transactions). 11 | 12 | There is a bitcoin-devwiki page https://github.com/bitcoin-core/bitcoin-devwiki/wiki/Mempool-and-mining[Mempool and mining^] which includes some additional mempool philosophy. 13 | 14 | James O'Beirne has https://github.com/jamesob/mempool.work/blob/master/README.md[written] a comprehensive overview of the current challenges and work in mempool design. 15 | It "documents the existing design, failures, and vulnerabilities of the mempool as well as some proposals that exist to remedy the shortcomings." 16 | 17 | == Mempool policy goals 18 | 19 | The documentation subfolder https://github.com/bitcoin/bitcoin/tree/master/doc/policy[doc/policy^] contains up-to-date information on **some**, but not all, of the current mempool policy rules. 20 | 21 | -------------------------------------------------------------------------------- /mempool-terminology.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Mempool terminology 2 | :page-nav_order: 0 3 | :page-parent: Mempool 4 | == Mempool terminology 5 | 6 | Ancestor(s):: One or more "parent" transactions which must be confirmed **before** the current transaction. + 7 | The ancestor transaction(s) _create_ outputs which are depended on by the current transaction. 8 | Descendant(s):: One or more "child" transactions which must be confirmed **after** the current transaction. + 9 | The descendant transaction(s) _depend_ on outputs from the current transaction. 10 | Orphan:: A transaction with missing ancestors. 11 | 12 | TIP: When _ancestor_ and _descendant_ are encountered in the codebase, they refer specifically to other **in-mempool** transactions. 13 | 14 | TIP: Ancestors and descendants can be confirmed in the same block but they must be in the correct order within the list of `transactions` for the block to be valid. 15 | -------------------------------------------------------------------------------- /mempool-tx-format.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Transaction format in the mempool 2 | :page-nav_order: 50 3 | :page-parent: Mempool 4 | [[mempool_tx_format]] 5 | == Transaction format in the mempool 6 | 7 | A `CTXMemPoolEntry` describes a mempool entry (i.e. transaction) in the mempool. 8 | It stores not only transaction information, but also pre-computed information about ancestors. 9 | 10 | .txmempool.h 11 | [source,cpp,options=nowrap] 12 | ---- 13 | 14 | class CTxMemPoolEntry 15 | { 16 | public: 17 | typedef std::reference_wrapper CTxMemPoolEntryRef; 18 | // two aliases, should the types ever diverge 19 | typedef std::set Parents; 20 | typedef std::set Children; 21 | 22 | private: 23 | const CTransactionRef tx; 24 | mutable Parents m_parents; 25 | mutable Children m_children; 26 | const CAmount nFee; //!< Cached to avoid expensive parent-transaction lookups 27 | const size_t nTxWeight; //!< ... and avoid recomputing tx weight (also used for GetTxSize()) 28 | const size_t nUsageSize; //!< ... and total memory usage 29 | const int64_t nTime; //!< Local time when entering the mempool 30 | const unsigned int entryHeight; //!< Chain height when entering the mempool 31 | const bool spendsCoinbase; //!< keep track of transactions that spend a coinbase 32 | const int64_t sigOpCost; //!< Total sigop cost 33 | int64_t feeDelta; //!< Used for determining the priority of the transaction for mining in a block 34 | LockPoints lockPoints; //!< Track the height and time at which tx was final 35 | 36 | // Information about descendants of this transaction that are in the 37 | // mempool; if we remove this transaction we must remove all of these 38 | // descendants as well. 39 | uint64_t nCountWithDescendants; //!< number of descendant transactions 40 | uint64_t nSizeWithDescendants; //!< ... and size 41 | CAmount nModFeesWithDescendants; //!< ... and total fees (all including us) 42 | 43 | // Analogous statistics for ancestor transactions 44 | uint64_t nCountWithAncestors; 45 | uint64_t nSizeWithAncestors; 46 | CAmount nModFeesWithAncestors; 47 | int64_t nSigOpCostWithAncestors; 48 | 49 | // ... 50 | ---- 51 | 52 | The advantage to having pre-computed data on descendants and ancestors stored with each transaction in the mempool is that operations involving adding and removing transactions can be performed faster. 53 | When a transaction is added to the mempool we must update the descendant data for all ancestor ``CTxMemPoolEntry``'s. 54 | Conversely if a transaction is removed from the mempool, we must also remove all of its descendants. 55 | A particular area where speed can be critical is in block template assembly. 56 | 57 | TIP: Some of this extra transaction metadata counts towards the mempool's maximum size, therefore a default mempool of 300MB will contain less than 300MB of serialized transactions. 58 | 59 | == Mapping transactions in the mempool 60 | 61 | A lot of how fee-maximizing block templates can be swiftly generated from chains of potentially-complex interlinked and dependant transactions comes down to ``CTxMemPool``'s `boost::multi_index` `mapTx`, which is able to natively store transactions in an index against multiple criteria as described in the https://www.boost.org/doc/libs/1_68_0/libs/multi_index/doc/index.html[documentation^] and code comments: 62 | 63 | .txmempool.h#CTxMemPool 64 | [source,cpp,options=nowrap] 65 | ---- 66 | 67 | /* 68 | * mapTx is a boost::multi_index that sorts the mempool on 5 criteria: 69 | * - transaction hash (txid) 70 | * - witness-transaction hash (wtxid) 71 | * - descendant feerate [we use max(feerate of tx, feerate of tx with all descendants)] 72 | * - time in mempool 73 | * - ancestor feerate [we use min(feerate of tx, feerate of tx with all unconfirmed ancestors)] 74 | */ 75 | 76 | ---- 77 | 78 | The index has 5 sort fields: the default, and tagged fields `index_by_wtxid`, `descendant_score`, `entry_time` and `ancestor_score`: 79 | 80 | . The default, and untagged, sort field of the index, which is using the https://www.boost.org/doc/libs/1_62_0/libs/multi_index/doc/reference/hash_indices.html#unique_non_unique[hashed_unique^] sort; hashing the `txid` using Bitcoin Core's implementation of the SipHash hasher for txids. + 81 | This is used when adding and removing transactions from the mempool, requesting and looking up mempool transactions (by txid) and checking whether RBF is enabled. 82 | . `index_by_wtxid` is used when checking whether transactions received over the P2P network already exist in the mempool (via the `exists()` function). 83 | . `descendant_score` is used when trying to trim the mempool to size (via `TrimToSize()`). + 84 | In this case we want to keep parent (ancestor) transactions in the mempool who have high fee-paying children (descendants). 85 | . `entry_time` is used to calculate when transactions in the mempool should expire. 86 | . `ancestor_score` is used to create new block templates by selecting the most valuable effective-feerate transaction chains. 87 | 88 | -------------------------------------------------------------------------------- /mempool-unbroadcast.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Mempool unbroadcast set 2 | :page-nav_order: 40 3 | :page-parent: Mempool 4 | == Mempool unbroadcast set 5 | 6 | The mempool contains an "unbroadcast" set called `m_unbroadcast_txids`. 7 | As the name implies this stores the txids of transactions which are in our mempool but have not been verified as broadcast to the wider P2P network. 8 | This might occur for example if a transaction is submitted locally (e.g. from the wallet or RPC), but we are not yet connected to any peers. 9 | 10 | . When a transaction is submitted to the network via `BroadcastTransaction()` it is added to the unbroadcast set in the mempool, before `PeerManager` calls `RelayTransaction()` to attempt initial broadcast. 11 | . When a transaction is heard about from the P2P network (via `getdata` in response to an `INV`), the transaction will be https://github.com/bitcoin/bitcoin/blob/v23.0/src/net_processing.cpp#L2022[removed] from the unbroadcast set. 12 | + 13 | TIP: Transactions are also removed from the set on reorgs, new blocks arriving or if they've "expired" via `RemoveStaged()` 14 | 15 | `PeerManager` schedules `ReattemptInitialBroadcast()` to be run every 10 minutes. 16 | This function loops over the unbroadcast set and either attempts rebroadcast or removes the transaction from the unbroadcast set if it is no longer in our mempool. 17 | 18 | TIP: amiti wrote a https://gist.github.com/amitiuttarwar/b592ee410e1f02ac0d44fcbed4621dba[gist^] on her proposal to improve rebroadcast logic in Bitcoin Core. 19 | 20 | -------------------------------------------------------------------------------- /mempool.adoc: -------------------------------------------------------------------------------- 1 | = Mempool 2 | :page-nav_order: 60 3 | :page-has_children: true 4 | -------------------------------------------------------------------------------- /mermaid-config-pdf.json: -------------------------------------------------------------------------------- 1 | { 2 | "themeCSS": ".label foreignObject { font-size: 85%; overflow: visible; }", 3 | "puppeteerConfigFile": "puppeteer-config.json", 4 | "outputFormat": "png" 5 | } 6 | -------------------------------------------------------------------------------- /mermaid-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "themeCSS": ".label foreignObject { font-size: 85%; overflow: visible; }" 3 | } 4 | -------------------------------------------------------------------------------- /message-relay.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Message relay 2 | :page-nav_order: 60 3 | :page-parent: P2P 4 | [[message_relay]] 5 | == Message relay 6 | 7 | .Relay policy of different messages 8 | [cols="1,3,3"] 9 | |=== 10 | |Message type |Function |Who 11 | 12 | |Addresses 13 | |`PeerManagerImpl::RelayAddress()` 14 | a|Outbound peers & inbound peers who send an addr-related message but not block-relay-only peers 15 | 16 | Reachable addresses to 2 peers. Unreachable addresses randomly to 1 or 2 peers. 17 | 18 | |Transactions 19 | |`PeerManagerImpl::RelayTransaction()` 20 | |All connected peers 21 | 22 | |Blocks 23 | a|`PeerManagerImpl::UpdatedBlockTip()` 24 | 25 | `PeerManagerImpl::MaybeSendAddr()` 26 | |All connected peers 27 | 28 | |=== 29 | -------------------------------------------------------------------------------- /multiple-chains.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Multiple chains 2 | :page-nav_order: 100 3 | :page-parent: Consensus and Validation 4 | == Multiple chains 5 | 6 | TODO: Reorgs, undo data, `DisconnectBlock` 7 | 8 | Bitcoin nodes should ultimately converge in consensus on the most-work chain. 9 | Being able to track and monitor multiple chain (tips) concurrently is a key requirement for this to take place. 10 | There are a number of different states which the client must be able to handle: 11 | 12 | . A single, most-work chain being followed 13 | . Stale blocks learned about but not used 14 | . Full reorganisation from one chain tip to another 15 | 16 | `BlockManager` is tasked with maintaining a tree of all blocks learned about, along with their total work so that the most-work chain can be quickly determined. 17 | 18 | `CChainstate` (https://github.com/bitcoin/bitcoin/pull/24513[renamed^] to `Chainstate` in v24.0) is responsible for updating our local view of the best tip, including reading and writing blocks to disk, and updating the UTXO set. 19 | A single `BlockManager` is shared between all instances of `CChainState`. 20 | 21 | `ChainstateManager` is tasked with managing multiple ``CChainState``s. 22 | Currently just a "regular" IBD chainstate and an optional snapshot chainstate, which might in the future be used as part of the https://bitcoinops.org/en/topics/assumeutxo/[assumeUTXO^] project. 23 | 24 | When a new block is learned about (from `src/net_processing.cpp`) it will call into ``ChainstateManager``s `ProcessNewBlockHeaders` method to validate it. 25 | -------------------------------------------------------------------------------- /multiple-transactions.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Multiple transactions 2 | :page-nav_order: 10 3 | :page-parent: Transaction Validation 4 | :page-grand_parent: Consensus and Validation 5 | include::links-onepage.adoc[] 6 | [#multiple_transactions] 7 | === Multiple transactions (and packages) 8 | 9 | TODO: This section should start from `AcceptPackage()` and flow through from there, including `AcceptMultipleTransactions()` as a sub-section. 10 | 11 | It's possible to consider multiple transactions for validation together, via `AcceptMultipleTransactions()` found in _src/net_processing.cpp_. 12 | It's currently only available from tests (`test/tx_package_tests.cpp`) and the `testmempoolaccept` RPC (via `ProcessNewPackage()`), but the intention is for it to be available to packages received from the P2P network in the future. 13 | 14 | This validation flow has been created for usage with Package Mempool Accept, which glozow has written up in a https://gist.github.com/glozow/dc4e9d5c5b14ade7cdfac40f43adb18a[gist^] (https://archive.ph/Uhewe[archive^]). 15 | 16 | The flow here is similar to xref:single-transactions.adoc#accept_single_transaction[`AcceptSingleTransaction()`] in that we start by grabbing `cs_main` before initializing validation state and workspaces, however this time we use `PackageValidationState` and a vector of workspaces, `std::vector`. 17 | Each transaction therefore has it's own workspace but all transactions in the package share a single validation state. 18 | This aligns with the goal of either accepting or rejecting the entire package as a single entity. 19 | 20 | Next come two `for` loops over the vector of workspaces (i.e. transactions). 21 | The first performs the xref:input-checks.adoc#prechecks[`PreChecks()`], but this time also freeing up coins to be spent by other transactions in this package. 22 | This would not usually be possible (nor make sense) _within_ an `AcceptTransaction()` flow, but within a package we want to be able to validate transactions who use as inputs, other transactions not yet added to our mempool: 23 | 24 | [source,cpp,options=nowrap] 25 | ---- 26 | // Make the coins created by this transaction available for subsequent transactions in the 27 | // package to spend. Since we already checked conflicts in the package and we don't allow 28 | // replacements, we don't need to track the coins spent. Note that this logic will need to be 29 | // updated if package replace-by-fee is allowed in the future. 30 | assert(!args.m_allow_bip125_replacement); 31 | m_viewmempool.PackageAddTransaction(ws.m_ptx); 32 | ---- 33 | 34 | If the `PreChecks` do not fail, we call `m_viewmempool.PackageAddTransaction()` passing in the workspace. 35 | This adds the transaction to a map in our Mempool called `std::unordered_map m_temp_added;`, which is essentially a temporary cache somewhere in-between being validated and being fully added to the mempool. 36 | 37 | TODO: Fix after adding section on `AcceptPackage` 38 | 39 | After this first loop we perform `PackageMempoolChecks()` which first asserts that transactions are not already in the mempool, before checking the "PackageLimits". 40 | -------------------------------------------------------------------------------- /multiwallet.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Multiwallet 2 | :page-nav_order: 150 3 | :page-parent: Wallet 4 | == Multiwallet 5 | 6 | Work on the https://github.com/bitcoin/bitcoin/pull/8694[multiwallet project^] means that Bitcoin Core can now handle dynamic loading and unloading of multiple wallets while running. 7 | 8 | -------------------------------------------------------------------------------- /netgroupmanager.adoc: -------------------------------------------------------------------------------- 1 | :page-title: NetGroupManager 2 | :page-nav_order: 30 3 | :page-parent: P2P 4 | === NetGroupManager 5 | 6 | NetGroupManager is used to encapsulate all https://blog.bitmex.com/call-to-action-testing-and-improving-asmap/[asmap^] data and logic. 7 | It is setup by loading any provided asmap file passed during startup. 8 | 9 | ==== History 10 | 11 | - https://bitcoincore.reviews/16702[PR#16702] introduced asmap as part of Addrman. 12 | - https://github.com/bitcoin/bitcoin/pull/22910[PR#22910^] introduced NetGroupManager as a better way to access asmap data by both Addrman and CConnman. 13 | 14 | -------------------------------------------------------------------------------- /network-entropy.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Networking contribution to entropy 2 | :page-nav_order: 120 3 | :page-parent: P2P 4 | === Networking contribution to node RNG entropy 5 | 6 | Entropy for the RNG is often harvested from network connections: 7 | 8 | .src/net.cpp 9 | [source,cpp,options=nowrap] 10 | ---- 11 | net.cpp 12 | 488- 13 | 489: // We're making a new connection, harvest entropy from the time (and our peer count) 14 | 490- RandAddEvent((uint32_t)id); 15 | -- 16 | 743- 17 | 744: // We just received a message off the wire, harvest entropy from the time (and the message checksum) 18 | 745- RandAddEvent(ReadLE32(hash.begin())); 19 | -- 20 | 1160- 21 | 1161: // We received a new connection, harvest entropy from the time (and our peer count) 22 | 1162- RandAddEvent((uint32_t)id); 23 | ---- 24 | 25 | -------------------------------------------------------------------------------- /node-components.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Node components 2 | :page-nav_order: 20 3 | :page-parent: P2P 4 | :mermaid-puppeteer-config: ./puppeteer-config.json 5 | == Node P2P components 6 | 7 | .Node P2P components 8 | [mermaid,target=node-p2p-components,align="center"] 9 | .... 10 | flowchart LR 11 | Addrman 12 | Banman 13 | CConnman 14 | CTxMempool 15 | ChainstateManager 16 | NetGroupManager 17 | Peerman 18 | Asmap{{Asmap}} 19 | peers.dat{{peers.dat}} 20 | anchors.dat{{anchors.dat}} 21 | banlist.dat{{banlist.dat}} 22 | mempool.dat{{mempool.dat}} 23 | 24 | subgraph node 25 | node.netgroupman 26 | node.addrman 27 | node.banman 28 | node.connman 29 | node.mempool 30 | node.chainman 31 | node.peerman 32 | end 33 | 34 | Asmap -.-> NetGroupManager 35 | peers.dat -.-> Addrman 36 | Addrman --> node.addrman 37 | 38 | NetGroupManager --> Addrman 39 | NetGroupManager --> CConnman 40 | NetGroupManager --> node.netgroupman 41 | 42 | banlist.dat -.-> Banman 43 | Banman --> Peerman 44 | Banman --> node.banman 45 | 46 | Addrman --> CConnman 47 | anchors.dat -.-> CConnman 48 | CConnman --> node.connman 49 | 50 | mempool.dat -.-> CTxMempool 51 | CTxMempool --> Peerman 52 | CTxMempool --> node.mempool 53 | 54 | ChainstateManager --> node.chainman 55 | 56 | Addrman --> Peerman 57 | CConnman --> Peerman 58 | Peerman --> node.peerman 59 | 60 | .... 61 | 62 | -------------------------------------------------------------------------------- /overview.adoc: -------------------------------------------------------------------------------- 1 | = Overview and Development Process 2 | :page-nav_order: 0 3 | :page-has_children: true 4 | 5 | TIP: This section has been updated to Bitcoin Core @ https://github.com/bitcoin/bitcoin/tree/v23.0[v23.0^] 6 | -------------------------------------------------------------------------------- /p2p-design-philosophy.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Design philosophy 2 | :page-nav_order: 0 3 | :page-parent: P2P 4 | == Design philosophy 5 | 6 | The P2P design philosophy is outlined in the bitcoin devwiki article https://github.com/bitcoin-core/bitcoin-devwiki/wiki/P2P-Design-Philosophy[P2P Design Philosophy^]. 7 | A synopsis of the ideas can be found in the first few paragraphs: 8 | 9 | [quote] 10 | ____ 11 | For the Bitcoin network to remain in consensus, the network of nodes must not be partitioned. 12 | So for an individual node to remain in consensus with the network, it must have at least one connection to that network of peers that share its consensus rules. 13 | 14 | ... 15 | 16 | We can't rely on inbound peers to be honest, because they are initiated by others. 17 | It's impossible for us to know, for example, whether all our inbound peers are controlled by the same adversary. 18 | 19 | Therefore, in order to try to be connected to the honest network, we focus on having good outbound peers, as we get to choose who those are. 20 | ____ 21 | 22 | The document, which is worth reading in its entirely, continues by assuming the case that we don't have any inbound peers but also considering that any inbound peers we _do_ have shouldn't be able to interfere with the P2P logic proposed. 23 | 24 | == Design goals 25 | 26 | :amiti-attacking: https://btctranscripts.com/la-bitdevs/2020-04-16-amiti-uttarwar-attacking-bitcoin-core/ 27 | 28 | Amiti Uttarwar created a framework of 5 goals she sees for the P2P network. 29 | 30 | TLDR; We want valid messages to make it out to the network (**reliable**) in a reasonable amount of time (**timely**) and for nodes to be able to get onto the network and stay on the network of their own accord (**accesible**). 31 | These three values seem quite important for any peer-to-peer network to be successful but in Bitcoin we have two additional. 32 | **Privacy** because it is money and **upgradeability** because of the ethos of Bitcoin. 33 | 34 | . **Reliable**; if a node submits a valid message to the network it will eventually be delivered to all other nodes on the network. 35 | . **Timely**; each of the messages have to make it out in a reasonable amount of time. 36 | * Reasonable amount of time for a transaction is different than for a block and reasonable amount of time for a block to be propagated for a normal user versus a miner is very different as well. 37 | . **Accessible**; the requirement to be able to participate must be low. Also an adversary shouldn’t be able to keep a node off the network. 38 | * Currently it is still possible to run a full Bitcoin Core node on a Raspberry Pi which is a low barrier-to-entry. 39 | . **Private**; because it is money and fundamentally it comes down to the idea of not wanting to connect your real world identity with your onchain interactions. 40 | . **Upgradeable**; stems from the ethos that if a user decides to buy into the rule set at a specific point in time they should always be able to transact with the rule set they initially bought into. 41 | 42 | **Reliability** vs **Privacy** can seem at odds with one another as is really hard to design and achieve both of them at the same time. 43 | For example, **value long-lasting connections**, can help for reliable delivery but **comes against privacy**. 44 | **Dynamic connections** help maintain transaction privacy, but **comes against reliability**. 45 | Reliability is you want to tell everyone your message, but privacy is you don’t want them to know that it is your message. 46 | 47 | See the {amiti-attacking}[transcript^] for more detail on each of those points. 48 | 49 | -------------------------------------------------------------------------------- /p2p-encryption.adoc: -------------------------------------------------------------------------------- 1 | :page-title: P2P encryption 2 | :page-nav_order: 110 3 | :page-parent: P2P 4 | == P2P message encryption 5 | 6 | P2P messages are currently all unencrypted which can potentially open up vulnerabilities like: 7 | 8 | * Associated metadata in P2P messages may reveal private information. 9 | * Possibilities for attackers who control the routing infrastructure of the P2P network to censor P2P 10 | messages since P2P messages can be detected trivially - they always start with a fixed sequence of magic bytes. 11 | 12 | https://gist.github.com/dhruv/5b1275751bc98f3b64bcafce7876b489[BIP 324^] proposes a new Bitcoin P2P protocol which features transport encryption and slightly lower bandwidth usage. 13 | 14 | https://bip324.com[bip324.com^] contains a list of all the open PRs and great resources to understand the proposal. 15 | A visual explanation of how BIP 324 works can be found in this blog - https://github.com/stratospher/blogosphere/blob/main/bip324.md[How to encrypt the P2P protocol?^] 16 | 17 | -------------------------------------------------------------------------------- /p2p-exercises.adoc: -------------------------------------------------------------------------------- 1 | :page-title: P2P exercises 2 | :page-nav_order: 160 3 | :page-parent: P2P 4 | include:links-onepage.adoc 5 | == Exercises 6 | 7 | include::exercise-intro.adoc[] 8 | 9 | . Make manual connections 10 | - [ ] Add the following configuration options to a new Bitcoin Core node running on signet to have it start it with no connections: 11 | + 12 | [source,text] 13 | ---- 14 | signet=1 15 | dnsseed=0 16 | fixedseeds=0 17 | debug=addrman 18 | ---- 19 | - [ ] Find the (only!) Signet DNS seeder node (in the `SigNetParams` class starting with "seed") and using a terminal poll this seed node for an address to connect to. 20 | + 21 | [TIP] 22 | ==== 23 | You can use `dig` or `nslookup` to retrieve seeds from the DNS seeder from the DNS seeders. 24 | ==== 25 | + 26 | [CAUTION] 27 | ==== 28 | If you try this with the mainnet seeds you will need to consider which xref:bootstrapping.adoc#service_flags[service flags] the seeder advertises support for. 29 | For example, if a seed node advertises `x1` support this means they return IP addresses of nodes advertizing the `NODE_NETWORK` service flag. 30 | 31 | You could query these from sipa's mainnet seeder by prepending `x1` to the subdomain e.g. `nslookup x1.seeder.bitcoin.sipa.be` 32 | ==== 33 | - [ ] Check how many addresses are known to your node: `bitcoin-cli -signet getnodeaddresses 0 | jq length` 34 | - [ ] Using one of the addresses returned from the previous exercise, connect to this node using the `addnode` RPC. 35 | - [ ] Observe new addresses being received and connected to in the bitcoind terminal or _$DATADIR/debug.log_ file. 36 | - [ ] What dangers can there be in retrieving node addresses in this way? 37 | - [ ] Is this more of less safe than using the hardcoded seeds? Can you think of a better way to distribute seeds to new users? 38 | -------------------------------------------------------------------------------- /p2p-violations.adoc: -------------------------------------------------------------------------------- 1 | :page-title: P2P violations 2 | :page-nav_order: 140 3 | :page-parent: P2P 4 | [[p2p_violations]] 5 | == P2P violations 6 | 7 | Bitcoin Core has several options for how to treat peers that violate the rules of the P2P protocol: 8 | 9 | . Ignore the individual message, but continue processing other messages from that peer 10 | . Increment the peer's "misbehaviour" score, and punish the peer once its score goes above a certain amount 11 | . Disconnect from the peer 12 | . Disconnect from the peer and prevent any later connections from that peer's address (discouragement) 13 | 14 | Since https://github.com/bitcoin/bitcoin/pull/20079[PR#20079^] we now treat handshake misbehaviour like an unknown message 15 | 16 | -------------------------------------------------------------------------------- /p2p.adoc: -------------------------------------------------------------------------------- 1 | = P2P 2 | :page-nav_order: 50 3 | :page-has_children: true 4 | include::links-onepage.adoc[] 5 | 6 | TIP: This section has been updated to Bitcoin Core @ https://github.com/bitcoin/bitcoin/tree/v23.0[v23.0^] 7 | 8 | With bitcoin we are seeking to create a permissionless network in which anyone can make a bitcoin transaction. 9 | Anybody should be free and able to run a node and join the network. 10 | 11 | The Bitcoin P2P network serves 3 purposes: 12 | 13 | * xref:address-relay.adoc#address_relay[Gossiping addresses] of known reachable nodes on the network 14 | * xref:transaction-relay.adoc#transaction_relay[Relaying unconfirmed transactions] 15 | * xref:block-relay.adoc#block_relay[Propagating blocks] 16 | 17 | Although these three purposes share the same network, they have different design goals and properties. 18 | Transaction relay is optimized for a combination of redundancy/robustness to peer misbehaviour as well as bandwidth minimization, while block relay is optimized to minimize delay. 19 | 20 | -------------------------------------------------------------------------------- /package-relay.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Package relay 2 | :page-nav_order: 60 3 | :page-parent: Mempool 4 | == Package relay 5 | 6 | https://bitcoinops.org/en/topics/package-relay/[Package Relay^] is a long-discussed concept and, at the time of writing, is a work in progress in Bitcoin Core. 7 | A significant portion of the project involves changes to mempool validation, which glozow describes in her gist https://gist.github.com/glozow/dc4e9d5c5b14ade7cdfac40f43adb18a[Package mempool accept^]. 8 | 9 | https://github.com/bitcoin/bitcoin/pull/20833[PR#20833^] added the ability for mempool validation to assess a set of dependent transactions and enabled the `testmempoolaccept` RPC to support multiple transactions. 10 | 11 | https://github.com/bitcoin/bitcoin/pull/21800[PR#21800^] added the ability to analyse and limit the ancestor and descendant sets of packages in relation to the mempool. 12 | 13 | https://github.com/bitcoin/bitcoin/pull/22674[PR#22674^] defined child-with-unconfirmed-parents packages and enabled submission of such packages to the mempool. 14 | 15 | These PRs were also accompanied by several refactoring efforts: 16 | https://github.com/bitcoin/bitcoin/pull/21062[PR#21062^], 17 | https://github.com/bitcoin/bitcoin/pull/22796[PR#22796^], 18 | https://github.com/bitcoin/bitcoin/pull/22675[PR#22675^], 19 | https://github.com/bitcoin/bitcoin/pull/22855[PR#22855^], 20 | https://github.com/bitcoin/bitcoin/pull/23381[PR#23381^]. 21 | 22 | The document https://github.com/bitcoin/bitcoin/blob/master/doc/policy/packages.md[doc/policy/packages.md^] contains current information on the stated package acceptance rules. 23 | 24 | -------------------------------------------------------------------------------- /peer-state.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Peer state 2 | :page-nav_order: 130 3 | :page-parent: P2P 4 | == Peer state 5 | 6 | Peer state is divided into two types: 7 | 8 | * **Network/Connection state**; any low level stuff, sending/receiving bytes, keeping statistics, eviction logic, etc. 9 | * **Application state**; any data that is transmitted within P2P message payloads, and the processing of that data. 10 | Examples are tx inventory, addr gossiping, ping/pong processing. 11 | 12 | There are three main data structures that handle peer state: 13 | 14 | * `CNode` (defined in `net.h`, used by `m_nodes`(`CConnman`) and covered by `m_nodes_mutex`) is concerned with the **connection state** of the peer. 15 | * `CNodeState` (defined in `netprocessing.cpp`, used by `m_node_states`(`PeerManager`) and covered by `cs_main`) is concerned with the **application state** of the peer. 16 | ** It maintains validation-specific state about nodes, therefore guarded by `cs_main`. 17 | * `Peer` (defined in `netprocessing.cpp`, used by `m_peer_map`(`PeerManager`) and covered by `m_peer_mutex`) is concerned with the **application state** of the peer. 18 | ** It doesn't contain validation-critical data, therefore it is not guarded by `cs_main` 19 | 20 | However, there is still some _application state_ contained in `CNode` for historic reasons. 21 | https://github.com/bitcoin/bitcoin/issues/19398[Issue 19398^] outlines the process to eventually move this out of `CNode` as well as the reasoning behind the introduction of the `Peer` struct. 22 | 23 | -------------------------------------------------------------------------------- /pinning-attacks.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Pinning attacks 2 | :page-nav_order: 70 3 | :page-parent: Mempool 4 | == Pinning attacks 5 | 6 | glozow describes pinning attacks in her document https://github.com/glozow/bitcoin-notes/blob/master/pinning.md["Pinning zoo"^]. 7 | 8 | -------------------------------------------------------------------------------- /pr-maturation.adoc: -------------------------------------------------------------------------------- 1 | :page-title: PR Maturation 2 | :page-nav_order: 130 3 | :page-parent: Overview and Development Process 4 | ==== ACK / NACK 5 | 6 | If you are communicating on an Issue or PR, you might be met with "ACK"s and "NACK"s (or even "approach (N)ACK" or similar). 7 | ACK, or "acknowledge" generally means that the commenter approves with what is being discussed previously. 8 | NACK means they tend to not approve. 9 | 10 | What should you do if your PR is met with NACKs or a mixture of ACKs and NACKs? 11 | Again there are no hard rules but generally you should try to consider all feedback as constructive criticism. 12 | This can feel hard when veteran contributors appear to drop by and with a single "NACK" shoot down your idea, but in reality it presents a good moment to pause and reflect on _why_ someone is not agreeing with the path forward you have presented. 13 | 14 | Although there are again no hard "rules" or "measurement" systems regarding (N)ACKs, maintainers -- who's job it is to measure whether a change has consensus before merging -- will often use their discretion to attribute more weight behind the (N)ACKs of contributors that they feel have a good understanding of the codebase in this area. 15 | 16 | This makes sense for many reasons, but lets imagine the extreme scenario where members of a Reddit/Twitter thread (or other group) all "https://www.dictionary.com/e/slang/brigading/[brigade]" a certain pull request on GitHub, filling it with tens or even hundreds of NACKs... 17 | In this scenario it makes sense for a maintainer to somewhat reduce the weighting of these NACKs vs the (N)ACKs of regular contributors: 18 | 19 | We are not sure which members of this brigade: 20 | 21 | * Know how to code and with what competency 22 | * Are familiar with the Bitcoin Core codebase 23 | * Understand the impact and repercussions of the change 24 | 25 | Whereas we can be more sure that regular contributors **and** those respondents who are providing additional rationale in addition to their (N)ACK, have some understanding of this nature. 26 | Therefore it makes sense that we should weight regular contributors' responses, and those with additional compelling rationale, more heavily than GitHub accounts created yesterday which reply with a single word (N)ACK. 27 | 28 | From this extreme example we can then use a sliding scale to the other extreme where, if a proven expert in this area is providing a lone (N)ACK to a change, that we should perhaps step back and consider this more carefully. 29 | 30 | Does this mean that your views as a new contributor are likely to be ignored? 31 | **No**!! 32 | However it might mean that you might like to include rationale in any ACK/NACK comments you leave on PRs, to give more credence to your views. 33 | 34 | When others are (N)ACK-ing your work, you should not feel discouraged because they have been around longer than you. 35 | If they have not left rationale for the comment, then you should ask them for it. 36 | If they have left rationale but you disagree, then you can politely state your reasons for disagreement. 37 | 38 | In terms of choosing a tone, the best thing to do it to participate in PR review for a while and observe the tone used in public when discussing changes. 39 | -------------------------------------------------------------------------------- /puppeteer-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "args": ["--no-sandbox"] 3 | } 4 | -------------------------------------------------------------------------------- /qml-gui.adoc: -------------------------------------------------------------------------------- 1 | :page-title: QML GUI 2 | :page-nav_order: 30 3 | :page-parent: GUI 4 | == QML GUI 5 | 6 | Since writing this documentation focus has been directed towards re-writing the Qt code leveraging the https://doc.qt.io/qt-5/qtqml-index.html[Qt QML^] framework. 7 | This will allow developers to create visually-superior, and easier to write and reason-about GUI code, whilst also lowering the barriers to entry for potential new developers who want to be able to focus on GUI code. 8 | 9 | The recommendation therefore is to familiarise yourself with Qt QML and review the current codebase for the latest developments. 10 | You can follow along with the latest QML work in the specific https://github.com/bitcoin-core/gui-qml/blob/main/src/qml/README.md[bitcoin-core/qml-gui^] repo. 11 | 12 | -------------------------------------------------------------------------------- /reading.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Contributor journeys 2 | :page-nav_order: 0 3 | :page-parent: Overview and Development Process 4 | == Contributor journeys 5 | 6 | Some Contributors have documented their journeys into the space which lets us learn about approaches they found useful, and also any pitfalls and things they found difficult along the way. 7 | 8 | * https://github.com/amitiuttarwar[Amiti Uttarwar^] - https://medium.com/@amitiu/onboarding-to-bitcoin-core-7c1a83b20365[Onboarding to Bitcoin Core^] 9 | * https://github.com/jonatack[Jon Atack^] - https://jonatack.github.io/articles/on-reviewing-and-helping-those-who-do-it[On Reviewing, and Helping Those Who Do It^] 10 | * https://github.com/jimmysong[Jimmy Song^] - https://medium.com/bitcoin-tech-talk/a-gentle-introduction-to-bitcoin-core-development-fdc95eaee6b8[A Gentle Introduction to Bitcoin Core Development^] 11 | 12 | == Decentralized development 13 | 14 | Olivia Lovenmark and Amiti Uttarwar describe in their https://blog.okcoin.com/2020/09/15/developing-bitcoin/[blog post^] "Developing Bitcoin", how changes to bitcoin follow the pathway from proposal to being merged into the software, and finally into voluntary adoption by users choosing to use the software. 15 | 16 | == Developer guidelines 17 | 18 | The Bitcoin Core project itself contains three documents of particular interest to Contributors: 19 | 20 | . https://github.com/bitcoin/bitcoin/tree/master/CONTRIBUTING.md[CONTRIBUTING.md^] -- How to get started contributing to the project. (Forking, creating branches, commit patches) 21 | . https://github.com/bitcoin/bitcoin/tree/master/doc/developer-notes.md[developer-notes.md^] -- Development guidelines, coding style etc. 22 | . https://github.com/bitcoin/bitcoin/blob/master/doc/productivity.md[productivity.md^] -- Many tips for improving developer productivity (ccache, reviewing code, refspecs, git diffs) 23 | . https://github.com/bitcoin/bitcoin/blob/master/test/README.md[test/README.md] -- Guidance on running the test suite 24 | 25 | TIP: Using `ccache` as described in _productivity.md_ above will speed up builds of Bitcoin Core dramatically. 26 | 27 | TIP: Setting up a ramdisk for the test suite as described in _test/README.md_ will speed up running the test suite dramatically. 28 | 29 | -------------------------------------------------------------------------------- /relay-preferences.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Relay preferences 2 | :page-nav_order: 100 3 | :page-parent: P2P 4 | === Notifying peers of relay preferences 5 | 6 | Currently, block-relay-only connections are established indirectly: 7 | 8 | * When making an outbound block-relay-only connection, a node sets the boolean flag `fRelay` in the version message to `false`. 9 | * `fRelay` (introduced in the context of https://github.com/bitcoin/bips/blob/master/bip-0037.mediawiki[BIP 37^]) does not imply that transactions cannot be sent for the entire duration of the connection - in its original use case with BIP37, relay of transactions can be activated later on. 10 | * `fRelay=false` is also used in *-blocksonly* mode, a low-bandwidth option in which a node does not want to receive transactions from *any peer*, but does participate in address relay. 11 | 12 | Therefore, nodes currently don't have a notion which of their incoming peers see the connection as block-relay-only and don't have any logic attached to it. 13 | 14 | https://github.com/bitcoin/bitcoin/pull/20726[PR#20726^], accompanied by the new BIP proposal https://github.com/sdaftuar/bips/blob/2021-02-bip338-fixups/bip-0338.mediawiki[BIP 338^], introduces the new p2p message `disabletx` for block-relay-only connections, which makes it explicit that no messages related to transaction relay should ever be exchanged over the duration of the connection. 15 | 16 | -------------------------------------------------------------------------------- /reproducible.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Reproducible builds 2 | :page-nav_order: 150 3 | :page-parent: Overview and Development Process 4 | == Reproducible Guix builds 5 | 6 | Bitcoin Core uses the https://guix.gnu.org/[Guix^] package manager to achieve reproducible builds. 7 | Carl Dong gave an introduction to GUIX via a https://btctranscripts.com/breaking-bitcoin/2019/bitcoin-build-system/[remote talk^] in 2019, and also discussed it further on a ChainCode https://btctranscripts.com/chaincode-labs/chaincode-podcast/2020-11-30-carl-dong-reproducible-builds/[podcast^] episode. 8 | 9 | There are official https://github.com/bitcoin/bitcoin/blob/master/contrib/guix/README.md[instructions^] on how to run a Guix build in the Bitcoin Core repo which you should certainly follow for your first build, but once you have managed to set up the Guix environment (along with e.g. MacOS SDK), hebasto provides a more concise workflow for subsequent or repeated builds in his https://gist.github.com/hebasto/7293726cbfcd0b58e1cfd5418316cee3[gist^]. 10 | 11 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | notebook 2 | -------------------------------------------------------------------------------- /responsible-disclosure.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Responsible disclosure 2 | :page-nav_order: 110 3 | :page-parent: Consensus and Validation 4 | == Responsible Disclosure 5 | 6 | Bitcoin Core has a defined process for reporting security vulnerabilities via it's responsible disclosure process. 7 | This is detailed in https://github.com/bitcoin/bitcoin/blob/master/SECURITY.md[SECURITY.md^]. 8 | 9 | Bugs which would need to be disclosed by following this process are generally those which could result in a consensus-failure, theft of funds, or creation of additional supply tokens (new coin issuance). 10 | If bugs of this nature are posted publicly then inevitably one or more persons will try to enact them, possibly causing severe harm or loss to one or many people. 11 | 12 | If you would like to learn more about the responsible disclosure process and why it's so important for Bitcoin Core, you can read the following: 13 | 14 | . https://medium.com/mit-media-lab-digital-currency-initiative/http-coryfields-com-cash-48a99b85aad4[Responsible disclosure in the era of cryptocurrencies^] 15 | . https://cacm.acm.org/magazines/2020/10/247597-responsible-vulnerability-disclosure-in-cryptocurrencies/fulltext[Responsible Vulnerability Disclosure in Cryptocurrencies^] 16 | -------------------------------------------------------------------------------- /reviewing.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Reviewing code 2 | :page-nav_order: 20 3 | :page-parent: Overview and Development Process 4 | === Reviewing code 5 | 6 | Jon Atack provides a guide to reviewing a Bitcoin Core PR in his article https://jonatack.github.io/articles/how-to-review-pull-requests-in-bitcoin-core[How To Review Pull Requests in Bitcoin Core^]. 7 | 8 | // Archived @ https://archive.is/MyohE 9 | Gloria Zhao's https://github.com/glozow/bitcoin-notes/blob/master/review-checklist.md[review checklist^] details what a "good" review might look like, along with some examples of what she personally considers good reviews. 10 | In addition to this, it details how potential Reviewers can approach a new PR they have chosen to review, along with the sorts of questions they should be asking (and answering) in order to provide a meaningful review. 11 | Some examples of the subject areas Gloria covers include the PR's subject area, motivation, downsides, approach, security and privacy risks, implementation of the idea, performance impact, concurrency footguns, tests and documentation needed. 12 | -------------------------------------------------------------------------------- /robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Disallow: /bin/ 3 | -------------------------------------------------------------------------------- /rpc.adoc: -------------------------------------------------------------------------------- 1 | = RPC / REST / ZMQ 2 | :page-nav_order: 90 3 | 4 | TIP: This section has been updated to Bitcoin Core @ https://github.com/bitcoin/bitcoin/tree/v24.0.1[v24.0.1^] 5 | 6 | == Adding new RPCs 7 | 8 | When trying to expose new information to users there are generally two possible approaches for developers to consider: 9 | 10 | . Add a new server-side RPC which directly delivers the new data 11 | . Create a new function on the client-side (e.g. the cli tool `bitcoin-cli`) which calls one or more existing RPCs and manipulates the results into the required format. 12 | 13 | If the data is not available from existing RPCs then option 1) must be taken. 14 | However if option 2) is available then this is the preferred first choice. 15 | This is for the following reasons: 16 | 17 | * Minimalistic approach: put client-side functionality into the client, not the server 18 | ** Adding server-side increases maintenance burden 19 | * Client-side functionality does not have to worry about API stability (as the RPCs do) 20 | * Functions can more easily start client side and migrate to server-side if heavily used, than visa-versa 21 | 22 | There may be other considerations though too: 23 | 24 | * If this functionality might be wanted in multiple clients, e.g. `bitcoin-cli` and the GUI `bitcoin-qt`, then rather than making two implementations in two clients it may make sense to add a single server-side function 25 | * Doing expensive computations on the client side when an inexpensive pathway is available server-side 26 | 27 | Ultimately there is no "correct" answer for all cases, but considering some of the above before implementing one way or another 28 | 29 | == HTTP Server 30 | 31 | Bitcoin Core's https://github.com/bitcoin/bitcoin/blob/v24.0.1/src/httpserver.cpp#L138-L139[HTTP server] is responsible for handling both RPC and REST requests, but not ZMQ. 32 | Since https://github.com/bitcoin/bitcoin/pull/5677[PR#5677] the server is based on https://libevent.org/[libevent2]. 33 | Libevent is a general purpose event notification library, but is used in Bitcoin Core specifically for HTTP requests (which it supports natively). 34 | 35 | Much (not all) of the libevent interface is hidden behind wrappers. 36 | For example, https://github.com/bitcoin/bitcoin/blob/v24.0.1/src/httpserver.h#L56[`HTTPRequest`] wraps `evhttp_request` and https://github.com/bitcoin/bitcoin/blob/v24.0.1/src/httpserver.h#L154[`HTTPEvent`] wraps https://libevent.org/doc/structevent__base.html[`event_base`]. 37 | 38 | The relevant workflow for how (for example) an RPC request is handled is roughly as follows: 39 | 40 | . The HTTP server receives an RPC command from a caller, creates an `evhttp_request` object and passes its pointer to `http_request_cb()` (this step is completely handled by libevent). 41 | . An `HTTPWorkItem` is https://github.com/bitcoin/bitcoin/blob/v24.0.1/src/httpserver.cpp#L264[created], containing the `evhttp_request` (wrapped in `HTTPRequest hreq`) as well as the path and reference to the handler function (which contains the business logic to be executed to deal with the request). 42 | ** There are https://github.com/bitcoin/bitcoin/blob/v24.0.1/src/httprpc.cpp#L301-L304[2 handlers] for RPCs. 43 | ** There are https://github.com/bitcoin/bitcoin/blob/v24.0.1/src/rest.cpp#L931-L943[12 handlers] for REST. 44 | . The `HTTPWorkItem` https://github.com/bitcoin/bitcoin/blob/v24.0.1/src/httpserver.cpp#L266[is put on the global] `WorkQueue g_work_queue`, which https://github.com/bitcoin/bitcoin/blob/v24.0.1/src/httpserver.cpp#L338-L344[is processed] by multiple worker threads asynchronously. 45 | . When the handler function of a `HTTPWorkItem` completes successfully, it calls https://github.com/bitcoin/bitcoin/blob/v24.0.1/src/httprpc.cpp#L230[`HTTPRequest::WriteReply()`], which triggers the libevent function https://libevent.org/doc/http_8h.html#a0a77d07263e20733a7562dcd576ad721[`evhttp_send_reply()`], which in turn returns a response to the caller and destroys the `evhttp_request` object. 46 | 47 | Endpoints are registered to the HTTP server by calling `RegisterHTTPHandler()`, such as e.g. in https://github.com/bitcoin/bitcoin/blob/v24.0.1/src/httprpc.cpp#L301-L304[`StartHTTPRPC()`]. 48 | 49 | The HTTP server is initiated and started from `AppInitServers()`, and stopped from `Shutdown()`. 50 | 51 | `StartHTTPServer()` https://github.com/bitcoin/bitcoin/blob/v24.0.1/src/httpserver.cpp#L433[adds] a thread for each worker to `g_thread_http_workers`. 52 | These threads will keep running until https://github.com/bitcoin/bitcoin/blob/v24.0.1/src/httpserver.cpp#L118[`WorkQueue::Interrupt()`] sets `running` to `false` and https://github.com/bitcoin/bitcoin/blob/v24.0.1/src/httpserver.cpp#L106[the queue is empty]. 53 | -------------------------------------------------------------------------------- /script-appendix.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Script Appendix 2 | :page-nav_order: 30 3 | :page-parent: Script 4 | == Appendix 5 | 6 | [[executing_scripts]] 7 | === Executing scripts 8 | 9 | Bitcoin differs from most other cryptocurrencies by not including the script with the unspent transaction output on the blockchain, only the scriptPubKey is publicly viewable until spending time. 10 | The practical effects of this are: 11 | 12 | * Users wishing to sign transactions which are locked using locking scripts require *two* pieces of information: 13 | .. The relevant private key(s) 14 | .. The `redeemScript`, i.e. the contract of the script itself. 15 | 16 | Scripts are executed by first evaluating the unlocking script, then evaluating the locking script on the same stack. 17 | If both of these steps result in a `1` (or any other non-zero value) being the only item on the stack, the script is verified as `true`. 18 | 19 | TODO: Not true exactly: 20 | https://bitcoin.stackexchange.com/questions/112439/how-can-the-genesis-block-contain-arbitrary-data-on-it-if-the-script-is-invalid 21 | 22 | If any of the following are true, the script will evaluate to `false`: 23 | 24 | * The final stack is empty 25 | * The top element on the stack is `0` 26 | * There is more than one element remaining on the stack 27 | * The script returns prematurely 28 | 29 | There are a number of other ways which scripts can fail TODO 30 | 31 | === Script inside of addresses 32 | 33 | Bitcoin addresses can be of a "script hash" type (P2SH, and now P2WSH). 34 | As the name implies a valid script is created before being hashed. 35 | This hash is then used to generate an address which coins can be sent to. 36 | Once coins have been received to this address a (redeem / witness) script which hashes to the same hash must be provided (`scriptPubKey`), along with a satisfactory `scriptSig` in order to authorize a new spend. 37 | 38 | The origins of this revolutionary (at the time) style of address are touched upon in this https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2022-May/020429.html[email^] from ZmnSCPxj. 39 | The general context of the email is recursive covenants. 40 | A portion of the email is quoted below for convenience: 41 | 42 | [quote, ZmnSCPxj] 43 | ____ 44 | Covenants were first expressed as a possibility, I believe, during discussions around P2SH. 45 | Basically, at the time, the problem was this: 46 | 47 | * Some receivers wanted to use k-of-n multisignature for improved security. 48 | * The only way to implement this, pre-P2SH, was by putting in the `scriptPubKey` all the public keys. 49 | * The sender is the one paying for the size of the `scriptPubKey`. 50 | * It was considered unfair that the sender is paying for the security of the receiver. 51 | 52 | Thus, `OP_EVAL` and the P2SH concept was conceived. 53 | Instead of the `scriptPubKey` containing the k-of-n multisignature, you create a separate script containing the public keys, then hash it, and the `scriptPubKey` would contain the hash of the script. 54 | By symmetry with the P2PKH template: 55 | 56 | [source] 57 | ---- 58 | OP_DUP OP_HASH160 OP_EQUALVERIFY OP_CHECKSIG 59 | ---- 60 | 61 | The P2SH template would be: 62 | 63 | [source] 64 | ---- 65 | OP_DUP OP_HASH160 OP_EQUALVERIFY OP_EVAL 66 | ---- 67 | 68 | `OP_EVAL` would take the stack top vector and treat it as a Bitcoin SCRIPT. 69 | 70 | It was then pointed out that `OP_EVAL` could be used to create recursive SCRIPTs by quining using `OP_CAT`. 71 | `OP_CAT` was already disabled by then, but people were talking about re-enabling it somehow by restricting the output size of `OP_CAT` to limit the O(2^N) behavior. 72 | 73 | Thus, since then, `OP_CAT` has been associated with ***recursive*** covenants (and people are now reluctant to re-enable it even with a limit on its output size, because recursive covenants). 74 | In particular, `OP_CAT` in combination with `OP_CHECKSIGFROMSTACK` and `OP_CHECKSIG`, you could get a deferred `OP_EVAL` and then use `OP_CAT` too to quine. 75 | 76 | Because of those concerns, the modern P2SH is now "just a template" with an implicit `OP_EVAL` of the `redeemScript`, but without any `OP_EVAL` being actually enabled. 77 | ____ 78 | 79 | For more details refer to https://github.com/bitcoin/bips/blob/master/bip-0016.mediawiki[BIP16^] which introduced P2SH addresses. 80 | 81 | -------------------------------------------------------------------------------- /script-cli.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Scripts on the CLI 2 | :page-nav_order: 20 3 | :page-parent: Script 4 | == Working with bitcoin script from the command line 5 | 6 | **** 7 | Blockchain Commons contains guides related to Bitcoin Script including: 8 | 9 | * https://github.com/BlockchainCommons/Learning-Bitcoin-from-the-Command-Line/blob/master/09_1_Understanding_the_Foundation_of_Transactions.md[Accessing scripts from transactions^] with `bitcoin-cli` 10 | * https://github.com/BlockchainCommons/Learning-Bitcoin-from-the-Command-Line/blob/master/09_2_Running_a_Bitcoin_Script.md[How scripts are evaluated^] 11 | * https://github.com/BlockchainCommons/Learning-Bitcoin-from-the-Command-Line/blob/master/09_3_Testing_a_Bitcoin_Script.md[Testing bitcoin scripts^] using `btcdeb` 12 | **** 13 | 14 | -------------------------------------------------------------------------------- /script.adoc: -------------------------------------------------------------------------------- 1 | = Script 2 | :page-nav_order: 70 3 | :page-has_children: true 4 | 5 | TIP: This section has been updated to Bitcoin Core @ https://github.com/bitcoin/bitcoin/tree/v23.0[v23.0^] 6 | 7 | == Script origins 8 | 9 | . New scripts are created when creating a new address. 10 | . Scripts can be learned about when we receive a new transaction from the P2P network or from a newly-connected block. 11 | . With Taproot there may be scripts in alternative Tapscript execution paths which nobody on the network will ever learn about. 12 | 13 | == Scripts in Bitcoin Core 14 | 15 | The primary script objects are found in _script.h_. 16 | An enum over all the permitted OPCODES, `enum opcodetype`. 17 | The `CScriptNum` class which handles arithmetic operations on integer ``CScriptNum``s, whether from a loose `int_64t` or from a second `CScriptNum` object. 18 | The `CScript` class which supports serializing data into scripts, along with many helper functions such as returning the script type. 19 | 20 | -------------------------------------------------------------------------------- /scriptpubkeymanagers.adoc: -------------------------------------------------------------------------------- 1 | :page-title: ScriptPubKeyManagers 2 | :page-nav_order: 110 3 | :page-parent: Wallet 4 | [id=scriptpubkeymanagers] 5 | == ScriptPubKeyManagers (SPKM) 6 | 7 | Each wallet contains one or more ``ScriptPubKeyManager``s which are derived from the https://github.com/bitcoin/bitcoin/blob/v23.0/src/wallet/scriptpubkeyman.h#L166[base^] SPKM class and are in control of storing the ``scriptPubkey``s managed by that wallet. 8 | 9 | **** 10 | "A wallet" in the general sense therefore becomes "a collection of ``ScriptPubKeyManager``s", which are each managing an address type. 11 | **** 12 | 13 | In the current implementation, this means that a default (descriptor) wallet consists of 8 ``ScriptPubKeyManager``s, one SPKM for each combination shown in the table <>. 14 | 15 | [id=descriptor-spkmans] 16 | .Descriptor wallet SPKMans 17 | [%autowidth.stretch] 18 | |=== 19 | 20 | |{nbsp} |LEGACY |P2SH-SEGWIT |BECH32 |BECH32M 21 | 22 | |Receive 23 | |✓ 24 | |✓ 25 | |✓ 26 | |✓ 27 | 28 | 29 | |Change 30 | |✓ 31 | |✓ 32 | |✓ 33 | |✓ 34 | 35 | |=== 36 | 37 | Here is the _descriptor_ wallet code fragment which sets up an SPKM for each `OUTPUT_TYPE`: 38 | 39 | .src/wallet/wallet.cpp#SetupDescriptorScriptPubKeyMans() 40 | [source,cpp,options=nowrap] 41 | ---- 42 | // ... 43 | 44 | for (bool internal : {false, true}) { 45 | for (OutputType t : OUTPUT_TYPES) { 46 | auto spk_manager = std::unique_ptr(new DescriptorScriptPubKeyMan(*this)); 47 | if (IsCrypted()) { 48 | if (IsLocked()) { 49 | throw std::runtime_error(std::string(__func__) + ": Wallet is locked, cannot setup new descriptors"); 50 | } 51 | if (!spk_manager->CheckDecryptionKey(vMasterKey) && !spk_manager->Encrypt(vMasterKey, nullptr)) { 52 | throw std::runtime_error(std::string(__func__) + ": Could not encrypt new descriptors"); 53 | } 54 | } 55 | spk_manager->SetupDescriptorGeneration(master_key, t, internal); 56 | uint256 id = spk_manager->GetID(); 57 | m_spk_managers[id] = std::move(spk_manager); 58 | AddActiveScriptPubKeyMan(id, t, internal); 59 | } 60 | } 61 | 62 | // ... 63 | ---- 64 | 65 | By contrast a Legacy wallet will set up a **single** SPKM which will then be _aliased_ to a SPKM for each of the 6 `LEGACY_OUTPUT_TYPES`: `LEGACY`, `P2SH-SEGWIT` and `BECH32`. 66 | This gives it the external appearance of 6 distinct SPKMans, when really it only has 1: 67 | 68 | .src/wallet/wallet.cpp#SetupLegacyScriptPubKeyMan() 69 | [source,cpp,options=nowrap] 70 | ---- 71 | // ... 72 | 73 | auto spk_manager = std::unique_ptr(new LegacyScriptPubKeyMan(*this)); 74 | for (const auto& type : LEGACY_OUTPUT_TYPES) { 75 | m_internal_spk_managers[type] = spk_manager.get(); 76 | m_external_spk_managers[type] = spk_manager.get(); 77 | } 78 | m_spk_managers[spk_manager->GetID()] = std::move(spk_manager); 79 | 80 | // ... 81 | ---- 82 | 83 | SPKMans are stored in maps inside a `CWallet` according to output type. 84 | "External" and "Internal" (SPKMans) refer to whether the addresses generated are designated for giving out "externally", i.e. for receiving new payments to, or for "internal", i.e. change addresses. 85 | 86 | Prior to https://github.com/bitcoin/bitcoin/commit/c729afd0a3b74a3943e4c359270beaf3e6ff8a7b[c729afd0^] the equivalent SPKM functionality (fetching new addresses and signing transactions) was contained within `CWallet` itself, now however is split out for better maintainability and upgradability properties as brought about by the https://github.com/bitcoin-core/bitcoin-devwiki/wiki/Wallet-Class-Structure-Changes[wallet box class structure changes^]. 87 | Therefore `CWallet` objects no longer handle keys and addresses. 88 | 89 | The change to a `CWallet` made up of (multiple) ``{Descriptor|Legacy}ScriptPubKeyMan``'s is also sometimes referred to as the "Wallet Box model", where each SPKM is thought of as a distinct "box" within the wallet, which can be called upon to perform new address generation and signing functions. 90 | -------------------------------------------------------------------------------- /settings.adoc: -------------------------------------------------------------------------------- 1 | // Position toc on left and only display 4 deep 2 | :toc: 3 | :toclevels: 3 4 | :toc: left 5 | :source-language: cpp 6 | :source-highlighter: rouge 7 | :rouge-style: github 8 | // Use fancy icons for admonition boxes 9 | :icons: font 10 | // Add section links to HTML 11 | :sectlinks: 12 | // Set default imagesdir 13 | :imagesdir: images 14 | :docinfo: shared 15 | // mermaid.js config 16 | :mermaid-config: mermaid-config.json 17 | :mermaid-format: svg 18 | :mermaid-puppeteer-config: puppeteer-config.json 19 | -------------------------------------------------------------------------------- /source-organization.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Source code organization 2 | :page-nav_order: 60 3 | :page-parent: Architecture 4 | == Source code organization 5 | 6 | Issue https://github.com/bitcoin/bitcoin/issues/15732[#15732^] describes how the Bitcoin Core project is striving to organize libraries and their associated source code, copied below for convenience: 7 | 8 | > Here is how I am thinking about the organization: 9 | > 10 | > * `libbitcoin_server.a`, `libbitcoin_wallet.a`, and `libbitcoinqt.a` should all be terminal dependencies. They should be able to depend on other symbols in other libraries, but no other libraries should depend on symbols in them (and they shouldn't depend on each other). 11 | > 12 | > * `libbitcoin_consensus.a` should be a standalone library that doesn't depend on symbols in other libraries mentioned here 13 | > 14 | > * `libbitcoin_common.a` and `libbitcoin_util.a` seem very interchangeable right now and mutually depend on each other. I think we should either merge them into one library, or create a new top-level `src/common/` directory complementing `src/util/`, and start to distinguish general purpose utility code (like argument parsing) from bitcoin-specific utility code (like formatting bip32 paths and using ChainParams). Both these libraries can be depended on by `libbitcoin_server.a`, `libbitcoin_wallet.a`, and `libbitcoinqt.a`, and they can depend on `libbitcoin_consensus.a`. If we want to split util and common up, as opposed to merging them together, then util shouldn't depend on libconsensus, but common should. 15 | > 16 | > 17 | > Over time, I think it'd be nice if source code organization reflected library organization . I think it'd be nice if all `libbitcoin_util` source files lived in `src/util`, all `libbitcoin_consensus.a` source files lived in `src/consensus`, and all `libbitcoin_server.a` code lived in `src/node` (and maybe the library was called `libbitcoin_node.a`). 18 | 19 | You can track the progress of these changes by following links from the issue to associated PRs. 20 | 21 | The `libbitcoin-kernel` https://github.com/bitcoin/bitcoin/issues/24303[project^] will provide further clean-ups and improvements in this area. 22 | 23 | If you want to explore for yourself which sources certain libraries require on the current codebase, you can open the file `src/Makefile.am` and search for `_SOURCES`. 24 | -------------------------------------------------------------------------------- /starting-ideas.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Getting started with development 2 | :page-nav_order: 110 3 | :page-parent: Overview and Development Process 4 | === Getting started with development 5 | 6 | One of the roles most in-demand from the project is that of code review, and in fact this is also one of the best ways of getting familiarized with the codebase too! 7 | Reviewing a few PRs and adding your review comments to the PR on GitHub can be really valuable for you, the PR author and the bitcoin community. 8 | This https://testing.googleblog.com/2018/05/code-health-understanding-code-in-review.html[Google Code Health^] blog post gives some good advice on how to go about code review and getting past "feeling that you're not as smart as the programmer who wrote the change". 9 | If you're going to ask some questions as part of review, try and keep questions https://testing.googleblog.com/2019/11/code-health-respectful-reviews-useful.html[respectful^]. 10 | 11 | There is also a Bitcoin Core PR https://bitcoincore.reviews/[Review Club^] held weekly on IRC which provides an ideal entry point into the Bitcoin Core codebase. 12 | A PR is selected, questions on the PR are provided beforehand to be discussed on irc.libera.chat #bitcoin-core-pr-reviews IRC room and a host will lead discussion around the changes. 13 | 14 | Aside from review, there are 3 main avenues which might lead you to submitting your *own* PR to the repository: 15 | 16 | . Finding a `good first issue`, as tagged in the https://github.com/bitcoin/bitcoin/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22[issue tracker^] 17 | . Fixing a bug 18 | . Adding a new feature (that you want for yourself?) 19 | 20 | Choosing a "good first issue" from an area of the codebase that seems interesting to you is often a good approach. 21 | This is because these issues have been somewhat implicitly "concept ACKed" by other Contributors as "something that is likely worth someone working on". 22 | Don't confuse this for meaning that if you work on it that it is certain to be merged though. 23 | 24 | If you don't have a bug fix or new feature in mind and you're struggling to find a good first issue which looks suitable for you, don't panic. 25 | Instead keep reviewing other Contributors' PRs to continue improving your understanding of the process (and the codebase) while you watch the Issue tracker for something which you like the look of. 26 | 27 | When you've decided what to work on it's time to take a look at the current behaviour of that part of the code and perhaps more importantly, try to understand _why_ this was originally implemented in this way. 28 | This process of codebase "archaeology" will prove invaluable in the future when you are trying to learn about other parts of the codebase on your own. 29 | -------------------------------------------------------------------------------- /stats.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Project statistics 2 | :page-nav_order: 190 3 | :page-parent: Overview and Development Process 4 | == Project stats 5 | 6 | .Bitcoin Core @ v24.0.1 7 | [source,text] 8 | ---- 9 | =============================================================================== 10 | Language Files Lines Code Comments Blanks 11 | =============================================================================== 12 | GNU Style Assembly 1 913 742 96 75 13 | Autoconf 23 3530 1096 1727 707 14 | Automake 5 1803 1505 85 213 15 | BASH 10 1772 1100 438 234 16 | Batch 1 1 1 0 0 17 | C 22 37994 35681 1183 1130 18 | C Header 481 72043 43968 17682 10393 19 | CMake 3 901 706 86 109 20 | C++ 687 197249 153482 20132 23635 21 | Dockerfile 2 43 32 5 6 22 | HEX 29 576 576 0 0 23 | Java 1 23 18 0 5 24 | JSON 94 7968 7630 0 338 25 | Makefile 51 2355 1823 198 334 26 | MSBuild 2 88 87 0 1 27 | Objective-C++ 3 186 134 20 32 28 | Prolog 2 22 16 0 6 29 | Python 298 66473 48859 7598 10016 30 | Scheme 1 638 577 29 32 31 | Shell 50 2612 1745 535 332 32 | SVG 20 720 697 15 8 33 | Plain Text 6 1125 0 1113 12 34 | TypeScript 98 228893 228831 0 62 35 | Visual Studio Pro| 16 956 940 0 16 36 | Visual Studio Sol| 1 162 162 0 0 37 | XML 2 53 47 0 6 38 | ------------------------------------------------------------------------------- 39 | HTML 2 401 382 0 19 40 | |- CSS 2 98 82 1 15 41 | (Total) 499 464 1 34 42 | ------------------------------------------------------------------------------- 43 | Markdown 192 33460 0 26721 6739 44 | |- BASH 16 206 173 12 21 45 | |- C 2 53 47 3 3 46 | |- C++ 3 345 267 54 24 47 | |- INI 1 7 6 0 1 48 | |- Lisp 1 13 13 0 0 49 | |- PowerShell 1 1 1 0 0 50 | |- Python 2 346 280 61 5 51 | |- Shell 3 21 17 1 3 52 | |- XML 1 4 4 0 0 53 | (Total) 34456 808 26852 6796 54 | =============================================================================== 55 | Total 2103 662960 530837 77663 54460 56 | =============================================================================== 57 | ---- 58 | 59 | Source: https://github.com/XAMPPRocky/tokei[tokei^] 60 | 61 | -------------------------------------------------------------------------------- /subtrees.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Subtrees and PIMPL 2 | :page-nav_order: 90 3 | :page-parent: Architecture 4 | include::links-onepage.adoc[] 5 | [[subtrees]] 6 | == Subtrees 7 | 8 | Several parts of the repository (LevelDB, crc32c, secp256k1 etc.) are subtrees of software maintained elsewhere. 9 | 10 | Some of these are maintained by active developers of Bitcoin Core, in which case changes should go directly upstream without being PRed directly against the project. 11 | They will be merged back in the next subtree merge. 12 | 13 | Others are external projects without a tight relationship with our project. 14 | 15 | There is a tool in `test/lint/git-subtree-check.sh` to check a subtree directory for consistency with its upstream repository. 16 | 17 | See the full https://github.com/bitcoin/bitcoin/blob/master/doc/developer-notes.md#subtrees[subtrees^] documentation for more information. 18 | 19 | == Implementation separation 20 | 21 | Many of the classes found throughout the codebase use the PIMPL technique to separate their implementation from the external representation. 22 | See xref:appendix.adoc#pimpl-technique[PIMPL technique] in the Appendix for more information. 23 | -------------------------------------------------------------------------------- /testing-p2p-changes.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Testing P2P changes 2 | :page-nav_order: 150 3 | :page-parent: P2P 4 | == Testing P2P changes 5 | 6 | It can be challenging to test P2P changes as tooling and functional tests are lacking. 7 | Often devs simply setup a new node with the patch and leave it for some time!? 8 | 9 | [TIP] 10 | ==== 11 | Is there fuzzing for P2P messages yet? 12 | ==== 13 | 14 | === Testing transaction and block relay under SegWit 15 | 16 | SegWit was a softfork defined in https://github.com/bitcoin/bips/blob/master/bip-0141.mediawiki[BIP 141^], with P2P changes defined in https://github.com/bitcoin/bips/blob/65529b12bb01b9f29717e1735ce4d472ef9d9fe7/bip-0144.mediawiki[BIP 144^]. 17 | 18 | SegWit was activated at block 481,824 in August 2017. 19 | Prior to activation, some very careful testing was carried out to verify different scenarios, for example: 20 | 21 | . How are transactions and blocks relayed between un-upgraded and upgraded nodes? 22 | . How do upgraded nodes find other upgraded nodes to connect to? 23 | . If a node is un-upgraded at activation time and subsequently upgrades, how does it ensure that the blocks that it previously validated (without segwit rules) are valid according to segwit rules? 24 | 25 | To enable this kind of testing, https://github.com/bitcoin/bitcoin/pull/8418[PR#8418^] made it possible to configure the segwit activation parameters using a `-bip9params` configuration option. 26 | That configuration option was later renamed to `-vbparams` in https://github.com/bitcoin/bitcoin/pull/10463[PR#10463^], and replaced with `-segwitheight` in https://github.com/bitcoin/bitcoin/pull/16060[PR#16060^]. 27 | 28 | Those options allowed starting a node which would never activate segwit by passing `-vbparams=segwit:0:0` (or later, `-segwitheight=-1`). 29 | This was used in the functional tests to test the node's behaviour across activation. 30 | 31 | The segwit mainnet activation was a one-time event. 32 | Now that segwit has been activated, those tests are no longer required. 33 | 34 | https://github.com/bitcoin/bitcoin/pull/21090[PR#21090^] removed the final tests that made use of `-segwitheight=0`. 35 | With those tests removed, the special casing for `-segwitheight=-1` behaviour can also be removed. 36 | That special casing impacted logic in _net_processing_, _validation_ and _mining_. 37 | -------------------------------------------------------------------------------- /testing-qt.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Testing QT 2 | :page-nav_order: 50 3 | :page-parent: GUI 4 | == Testing QT 5 | 6 | Currently, although several QT tests exist in _src/qt/test_, there is no good way to test QT changes except by hand. 7 | A good way to try and have QT code included in the test framework is to target having the RPC layer be a thin as possible, so more code can be re-used between RPC and GUI. 8 | -------------------------------------------------------------------------------- /testing.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Testing your contributions 2 | :page-nav_order: 40 3 | :page-parent: Overview and Development Process 4 | ==== Continuous integration 5 | 6 | When PRs are submitted against the primary Bitcoin Core repo a series of CI https://github.com/bitcoin/bitcoin/tree/v23.0/ci[tests^] will automatically be run. 7 | These include a series of linters and formatters such as `clang-format`, `flake8` and `shellcheck`. 8 | It's possible (and advised) to run these checks locally against any changes you make before you push them. 9 | 10 | In order to run the lints yourself you'll have to first make sure your python environment and system have the packages listed in the CI install https://github.com/bitcoin/bitcoin/blob/v23.0/ci/lint/04_install.sh[script]. 11 | You can then run a decent sub-set of the checks by running: 12 | 13 | [source,bash] 14 | ---- 15 | python test/lint/lint-circular-dependencies.py 16 | 17 | # requires requires 'flake8', 'mypy', 'pyzmq', 'codespell', 'vulture' 18 | python test/lint/lint-python.py 19 | 20 | python test/lint/lint-whitespace.py 21 | ---- 22 | 23 | Or you can run all checks with: 24 | 25 | [source,bash] 26 | ---- 27 | cd ./test/lint/test_runner/ && cargo fmt && cargo clippy && cargo run 28 | ---- 29 | 30 | NOTE: Previously these checks were shell scripts (`*.sh`), but they have now been migrated to python on master. 31 | + 32 | If you are following with tag v23.0 these may still exist as `*.sh`. 33 | 34 | Linting your changes reduces the chances of pushing them as a PR and then having them quickly being marked as failing CI. The GitHub PR page auto-updates the CI status. 35 | 36 | TIP: If you do fail a lint or any other CI check, force-pushing the fix to your branch will cancel the currently-running CI checks and restart them. 37 | 38 | ==== Build issues 39 | 40 | Some compile-time issues can be caused by an unclean build directory. 41 | The comments in https://github.com/bitcoin/bitcoin/issues/19330[issue 19330^] provide some clarifications and tips on how other Contributors clean their directories, as well as some ideas for shell aliases to boost productivity. 42 | -------------------------------------------------------------------------------- /tests-overview.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Tests overview 2 | :page-nav_order: 30 3 | :page-parent: Architecture 4 | :mermaid-puppeteer-config: ./puppeteer-config.json 5 | == Tests overview 6 | 7 | .Tests overview 8 | [%autowidth] 9 | |=== 10 | |Tool |Usage 11 | 12 | |unit tests 13 | |`make check` or `./src/test_bitcoin` 14 | 15 | |functional tests 16 | |`test/functional/test_runner.py` 17 | 18 | |lint checks 19 | |See the https://github.com/bitcoin/bitcoin/blob/master/test/lint/README.md#running-locally[documentation^] 20 | 21 | |fuzz 22 | |See the https://github.com/bitcoin/bitcoin/blob/master/doc/fuzzing.md[documentation^] 23 | 24 | |util tests 25 | |`test/util/test_runner.py` 26 | 27 | |=== 28 | 29 | Bitcoin Core is also introducing (functional) "stress tests" which challenge the program via interruptions and missing files to ensure that we fail gracefully, e.g. the tests introduced in https://github.com/bitcoin/bitcoin/pull/23289[PR#23289^]. 30 | 31 | === Test directory structure 32 | 33 | The following diagram gives a brief overview of how the tests are structured within the source directory. 34 | 35 | WARNING: This diagram is **not** exhaustive and includes simplifications. 36 | 37 | NOTE: dashed lines indicate optional components 38 | 39 | NOTE: The `fuzz_targets` themselves are located in the `test` folder, however the fuzz tests are run via the `test_runner` in src/test so we point fuzz to there. 40 | 41 | NOTE: `qa_assets` are found in a https://github.com/bitcoin-core/qa-assets[separate^] repo altogether, as they are quite large (~3.5GB repo size and ~13.4GB on clone). 42 | 43 | .Test directory Structure 44 | [mermaid, target=bitcoin-core-tests] 45 | .... 46 | flowchart LR 47 | TR1[test_runner] 48 | functional --> test 49 | lint --> test 50 | util --> test 51 | test_framework --> functional 52 | TR1 --> functional 53 | 54 | TR2["test_runner (fuzz)"] 55 | fuzz_targets --> fuzz 56 | fuzz --> src/test 57 | libFuzzer -.-> fuzz 58 | TR2 --> fuzz 59 | qa-assets -.-> fuzz 60 | unit --> src/test 61 | Boost --> unit 62 | test_bitcoin --> unit 63 | 64 | classDef types fill:green,color:white,stroke:green; 65 | class functional,lint,util,fuzz,unit types 66 | .... 67 | 68 | === Test coverage 69 | 70 | Bitcoin Core's test coverage reports can be found https://maflcko.github.io/b-c-cov/[here^]. 71 | 72 | === Mutation testing 73 | 74 | Bitcoin Core's mutation testing reports can be found https://bitcoincore.space/[here^]. 75 | 76 | -------------------------------------------------------------------------------- /threads.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Threads 2 | :page-nav_order: 40 3 | :page-parent: Architecture 4 | == Threads 5 | 6 | The `main()` function starts the main bitcoind process thread, usefully named `bitcoind`. 7 | All subsequent threads are currently started as children of the `bitcoind` thread, although this is not an explicit design requirement. 8 | 9 | The Bitcoin Core Developer https://github.com/bitcoin/bitcoin/blob/master/doc/developer-notes.md#threads=[docs^] contains a section on threads, which is summarised below in two tables, one for net threads, and one for other threads. 10 | 11 | .Non-net threads 12 | [%autowidth] 13 | |=== 14 | |Name |Function |Description 15 | 16 | |`bitcoind` 17 | |`main()` 18 | |Responsible for starting up and shutting down the application, and spawning all sub-threads 19 | 20 | |`b-loadblk` 21 | |`ThreadImport` 22 | |Loads blocks from `blk*.dat` files or `-loadblock=` on startup 23 | 24 | |`b-scriptch.x` 25 | |`ThreadScriptCheck` 26 | |Parallel script validation threads for transactions in blocks 27 | 28 | |`b-http` 29 | |`ThreadHTTP` 30 | |Libevent thread to listen for RPC and REST connections 31 | 32 | |`b-httpworker.x` 33 | |`StartHTTPServer` 34 | |HTTP worker threads. Threads to service RPC and REST requests 35 | 36 | |`b-txindex.x` 37 | |`ThreadSync` 38 | |Indexer threads. One thread per indexer 39 | 40 | |`b-scheduler` 41 | |`SchedulerThread` 42 | |Does asynchronous background tasks like dumping wallet contents, dumping `addrman` and running asynchronous `validationinterface` callbacks 43 | 44 | |`b-torcontrol` 45 | |`TorControlThread` 46 | |Libevent thread for tor connections 47 | 48 | |=== 49 | 50 | [[net_threads]] 51 | === Net threads 52 | 53 | .Net threads 54 | [%autowidth] 55 | |=== 56 | |Name |Function |Description 57 | 58 | |`b-msghand` 59 | |`ThreadMessageHandler` 60 | |Application level message handling (sending and receiving). Almost all `net_processing` and validation logic runs on this thread 61 | 62 | |`b-dnsseed` 63 | |`ThreadDNSAddressSeed` 64 | |Loads addresses of peers from the `ThreadDNS` 65 | 66 | |`b-upnp` 67 | |`ThreadMapPort` 68 | |Universal plug-and-play startup/shutdown 69 | 70 | |`b-net` 71 | |`ThreadSocketHandler` 72 | |Sends/Receives data from peers on port 8333 73 | 74 | |`b-addcon` 75 | |`ThreadOpenAddedConnections` 76 | |Opens network connections to added nodes 77 | 78 | |`b-opencon` 79 | |`ThreadOpenConnections` 80 | |Initiates new connections to peers 81 | 82 | |`b-i2paccept` 83 | |`ThreadI2PAcceptIncoming` 84 | |Listens for and accepts incoming I2P connections through the I2P SAM proxy 85 | 86 | |=== 87 | 88 | === Thread debugging 89 | 90 | In order to debug a multi-threaded application like bitcoind using gdb you will need to enable following child processes. 91 | Below is shown the contents of a file `threads.brk` which can be sourced into gdb using `source threads.brk`, before you start debugging bitcoind. 92 | The file also loads break points where new threads are spawned. 93 | 94 | .threads.brk 95 | [source,bash] 96 | ---- 97 | set follow-fork-mode child 98 | break node::ThreadImport 99 | break StartScriptCheckWorkerThreads 100 | break ThreadHTTP 101 | break StartHTTPServer 102 | break ThreadSync 103 | break SingleThreadedSchedulerClient 104 | break TorControlThread 105 | break ThreadMessageHandler 106 | break ThreadDNSAddressSeed 107 | break ThreadMapPort 108 | break ThreadSocketHandler 109 | break ThreadOpenAddedConnections 110 | break ThreadOpenConnections 111 | break ThreadI2PAcceptIncoming 112 | ---- 113 | 114 | -------------------------------------------------------------------------------- /transaction-creation.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Transaction creation and signing 2 | :page-nav_order: 50 3 | :page-parent: Wallet 4 | === Transaction creation 5 | 6 | Once the coins have been selected they are returned back to `CreateTransactionInternal()`, which will create the final transaction. 7 | 8 | Right now when we determine the change output, we don't use what `selectionResult` says the change output should be. 9 | What we actually do is make the tx with in? outputs and set the change amount to be the sum inputs-outputs, so the change amount includes the transaction fee. 10 | To get the correct change amount we now calculate the size of this after signing, we use dummysigner to add a dummy signature (74 0's and the correct script), and now we can calculate the correct fee. 11 | We reduce that fee from the change output amount, and if this now goes below *some threshold?* (the "cost of change" thing from BnB) or if it is dust we drop the change output and add it's value to the fee. 12 | 13 | So now we have an unsigned tx which we need to sign. 14 | 15 | === Signing 16 | 17 | We pass the tx to `CWallet::SignTransaction()` which will call `IsMine()` on each input to figure out which ScriptPubKeyMan (spkman) owns that input, then ask the spkman to fetch its `SigningProviders` to provide the signer which can sign the transaction, and return that to us. 18 | 19 | With PSBTs we have the `fillPSBT()` method in `CWallet` which calls `*ScriptPubKeyMan::fillPSBT()`. 20 | We do this because we can add previous UTXOs due to transaction tracking; the SPKM adds the scripts and key derivation paths and will then optionally sign. 21 | 22 | == Separation of wallet and node 23 | 24 | Both the `bitcoind` and `bitcoin-qt` programs use the same source code for the wallet component. 25 | `bitcoin-qt` is not therefore a gui frontend for `bitcoind` but a stand-alone binary which happens to share much of the same code. 26 | There has been discussion since at least as early as 2014 about https://github.com/bitcoin/bitcoin/issues/3882[splitting wallet code^] out from the rest of the codebase, however this has not been completed yet. 27 | 28 | The https://github.com/bitcoin-core/bitcoin-devwiki/wiki//Process-Separation[Process Separation^] project is tracking development working towards separating out node, wallet and GUI code even further. 29 | In the mean time developers have preferred to focus on improving the organisation of the (wallet) source code within the project and to focus on making wallet code more asynchronous and independent of node code, to avoid locking the node while wallet code-paths are executing. 30 | 31 | -------------------------------------------------------------------------------- /transaction-tracking.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Transaction tracking 2 | :page-nav_order: 20 3 | :page-parent: Wallet 4 | === Transaction tracking 5 | 6 | When we learn about a new block the `BlockConnected` signal is https://github.com/bitcoin/bitcoin/blob/v23.0/src/validation.cpp#L2940[fired^] after successful validation. 7 | This prompts the wallet to https://github.com/bitcoin/bitcoin/blob/v23.0/src/wallet/wallet.cpp#L1317-L1328[iterate^] all inputs and outputs, calling `IsMine()` on all of them. 8 | As part of the https://github.com/bitcoin/bitcoin/blob/v23.0/src/wallet/wallet.cpp#L1100[check^], we https://github.com/bitcoin/bitcoin/blob/v23.0/src/wallet/wallet.cpp#L1394-L1396[loop^] over the wallet's ``scriptPubkeyMan``s to check if any of the scripts belong to us. 9 | 10 | If a script does belong to us, it will be inserted into `mapWallet` along with some metadata related to the time. 11 | `mapWallet` contains all the transactions the wallet is interested in, including received and sent transactions. 12 | 13 | When we https://github.com/bitcoin/bitcoin/blob/v23.0/src/wallet/wallet.cpp#L237[load^] a wallet into memory, we iterate all `TxSpends`. 14 | `TxSpends` stores wallet transactions which were already spent and confirmed. 15 | 16 | Therefore, when the wallet needs to select coins to spend, it can select from the coins: 17 | 18 | `mapWallet - TxSpends - notMine` 19 | 20 | -------------------------------------------------------------------------------- /userspace-files.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Userspace files 2 | :page-nav_order: 70 3 | :page-parent: Architecture 4 | == Userspace files 5 | 6 | Bitcoin Core stores a number of files in its data directory ($DATADIR) at runtime. 7 | 8 | === Block and undo files 9 | 10 | $DATADIR/blocks/blk*.dat:: 11 | Stores raw network-format block data in order received. 12 | 13 | $DATADIR/blocks/rev*.dat:: 14 | Stores block "undo" data in order processed. 15 | + 16 | [quote,Pieter Wuille,'https://bitcoin.stackexchange.com/questions/11104/what-is-the-database-for?rq=1[stackexchange]'] 17 | ____ 18 | You can see blocks as 'patches' to the chain state (they consume some unspent outputs, and produce new ones), and see the undo data as reverse patches. They are necessary for rolling back the chainstate, which is necessary in case of reorganisations. 19 | ____ 20 | 21 | //// 22 | TODO: bitcoin.conf, config.json, debug.log ?? 23 | //// 24 | 25 | === Indexes 26 | 27 | With data from the raw block* and rev* files, various LevelDB indexes can be built. 28 | These indexes enable fast lookup of data without having to rescan the entire block chain on disk. 29 | 30 | Some of these databases are mandatory and some of them are optional and can be enabled using run-time configuration flags. 31 | 32 | Block Index:: 33 | Filesystem location of blocks + some metadata 34 | 35 | Chainstate Index:: 36 | All current UTXOs + some metadata 37 | 38 | Tx Index:: 39 | Filesystem location of all transactions by txid 40 | 41 | Block Filter Index:: 42 | https://github.com/bitcoin/bips/blob/master/bip-0157.mediawiki[BIP157] filters, hashes and headers 43 | 44 | Coinstats Index:: 45 | UTXO set https://github.com/bitcoin/bitcoin/blob/b3f866a8dfd652b6339b79124843e58bd0bf3013/src/index/coinstatsindex.h#L26-L37[statistics] 46 | 47 | [%autowidth] 48 | |=== 49 | |Name |Location |Optional |Class 50 | 51 | |Block Index 52 | |_$DATADIR/blocks/index_ 53 | |No 54 | |`BlockIndex` 55 | 56 | |Chainstate Index 57 | |_$DATADIR/chainstate_ 58 | |No 59 | |`Chainstate` 60 | 61 | |Tx Index 62 | |_$DATADIR/indexes/txindex_ 63 | |Yes 64 | |`TxIndex` 65 | 66 | |Block Filter Index 67 | |_$DATADIR/indexes/blockfilter/_ 68 | |Yes 69 | |`BlockFilterIndex` 70 | 71 | |Coinstats Index 72 | |_$DATADIR/indexes/coinstats_ 73 | |Yes 74 | |`CoinstatsIndex` 75 | 76 | |=== 77 | -------------------------------------------------------------------------------- /wallet-database.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Wallet Database 2 | :page-nav_order: 0 3 | :page-parent: Wallet 4 | :config: ./mermaid-config.json 5 | :mermaid-puppeteer-config: ./puppeteer-config.json 6 | === Wallet Database 7 | 8 | Wallets are stored on disk as databases, either using Berkeley Database (BDB) or sqlite format. 9 | 10 | TIP: The version of BDB we used for the wallet is unmaintained, so new wallets should prefer sqlite format 11 | 12 | The wallet is stored on disk as a Key Value store. 13 | 14 | .Wallet database 15 | [mermaid,target=wallet-database,align="center"] 16 | .... 17 | flowchart BT 18 | database["database\n(KV store)"] 19 | 20 | Transactions --> database 21 | Keys --> database 22 | Key_metadata --> database 23 | wallet_metadata --> database 24 | database -- load db --> CWallet 25 | .... 26 | 27 | These are some of the https://github.com/bitcoin/bitcoin/blob/master/src/wallet/walletdb.cpp#L30-L62[records^] which help us regenerate a descriptor wallet (populating a `DescriptorScriptPubKeyMan` (DSPKM)) from the database: 28 | 29 | [source,cpp,options=nowrap] 30 | ---- 31 | // walletdb.cpp 32 | const std::string WALLETDESCRIPTOR{"walletdescriptor"}; 33 | const std::string WALLETDESCRIPTORCACHE{"walletdescriptorcache"}; 34 | const std::string WALLETDESCRIPTORLHCACHE{"walletdescriptorlhcache"}; 35 | const std::string WALLETDESCRIPTORCKEY{"walletdescriptorckey"}; 36 | const std::string WALLETDESCRIPTORKEY{"walletdescriptorkey"}; 37 | ---- 38 | 39 | For Legacy wallets (populating a `LegacyScriptPubKeyMan` (LSPKM)) we use the records with `*.KEY` & `SCRIPT`. 40 | 41 | Wallet metadata may include a `tipLocator` -- the most recent tip -- and a wallet `version` which is used in database upgrades. 42 | 43 | To load the wallet we read the database by iterating the records and loading them to `CWallet`, using `ReadKeyValue()` to deserialize. 44 | 45 | .Loading wallet records from the database 46 | [cols="2,3"] 47 | |=== 48 | |Record |Load point 49 | 50 | |`DBKeys::TX` 51 | |(Bitcoin) transactions end up in `mapWallet` via the call to `pwallet->LoadToWallet(hash, fill_wtx)` 52 | 53 | |`DBKeys::KEY` 54 | |Keys for legacy wallets are loaded into `CKey` or `Key`, then read into the appropriate SPKM (or one is created and keys added to it) using `pwallet->GetOrCreateLegacyScriptPubKeyMan()`. 55 | 56 | a|`DBKeys::WALLETDESCRIPTOR` + 57 | `DBKeys::WALLETDESCRIPTORCACHE` + 58 | `DBKeys::WALLETDESCRIPTORLHCACHE` + 59 | `DBKeys::WALLETDESCRIPTORKEY` + 60 | `DBKeys::WALLETDESCRIPTORCKEY` 61 | 62 | |Descriptor wallet information generally goes into `DescriptorScriptPubKeyMan`. 63 | 64 | a|`DBKeys::NAME` + 65 | `DBKeys::PURPOSE` 66 | 67 | |Addresses go into `m_address_book` 68 | 69 | |=== 70 | 71 | You can see where all the other DB records are deserialized to by examining the `ReadKeyValue()` https://github.com/bitcoin/bitcoin/blob/master/src/wallet/walletdb.cpp#L321-L746[function]. 72 | 73 | The various `*ScriptPubkeyMan` objects are all owned by the `CWallet` instance eventually, however `LegacyScriptPubKeyMan` is both created and owned by `CWallet`, whereas `DescriptorScriptPubKeyMan` is created externally to `CWallet` and only after loading exists in the `CWallet` context. 74 | 75 | Note that `TxSpends` is **not** tracked in the wallet database (and loaded at startup), but instead is rebuilt from scratch because it's fast to do so and we must reload every transaction anyway, so it's not much more work to regenerate `TxSpends` at the same time. 76 | //// 77 | TODO: Why do we reload every transaction anyway ^? 78 | //// 79 | 80 | === Key-type classes in the wallet 81 | 82 | There are a number of `Key` classes in the wallet code and keeping track of their functions can be confusing at times due to naming similarities. 83 | Below are listed some of these classes along with some primary functions. 84 | 85 | [id=wallet-key-types] 86 | **** 87 | [horizontal] 88 | `CKey`:: An encapsulated private key. Used for signing and deriving child keys. 89 | `CKeyID`:: A _reference_ to a `CKey` by the hash160 of its pubkey. Used for key lookups when fetching keys e.g. for signing. 90 | `CPrivKey`:: A serialized (OpenSSL format) private key with associated parameters. Used to read/write private keys to/from the database. 91 | `CPubKey`:: A public key. Used in many places. 92 | `CExtKey`:: An extended public key (includes private key and chaincode). Used for deriving BIP32 child keys. 93 | `CMasterKey`:: Contains an encryption salt `vchSalt` and a randomly generated encryption key `vchCryptedKey`. The `CMasterKey` object itself is what is encrypted by the user's passphrase and the inner `vchCryptedKey` is what is used to en/de-crypt the wallet keys. 94 | `CKeyingMaterial`:: Plain text which is to be encrypted or has been decrypted using the `CMasterKey`. 95 | `CKeyPool`:: A single key which has been taken from a ``CWallet``'s keypool for use. `CKeyPool` keys are stored in the wallet database. 96 | `CryptedKeyMap`:: A map of `CKeyID` to a pair of (`CPubKey` + an encrypted private key). Used to lookup keys (by `CKeyID`) when the wallet is encrypted. 97 | **** 98 | -------------------------------------------------------------------------------- /wallet-encryption.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Wallet Encryption 2 | :page-nav_order: 10 3 | :page-parent: Wallet 4 | include::links-onepage.adoc[] 5 | === Encryption 6 | 7 | There is encryption in the wallet code, but it is found within both `CWallet` and `*ScriptPubKeyMan` so is not yet well encapsulated. 8 | 9 | IMPORTANT: When encryption is enabled secret data must only ever reside in memory and should **never** be written to disk. 10 | 11 | When you unlock an encrypted wallet you can set a `timeout`. 12 | When the timeout expires secret data is deleted from memory, and the wallet "re-locked". 13 | 14 | ==== Decrypting the wallet 15 | 16 | As detailed in xref:wallet-database.adoc#wallet-key-types[Key types], the `CMasterKey.vchCryptedKey` is the actual secret key used to en/de-crypt the keys in the wallet. 17 | 18 | `CWallet` stores a `CMasterKey`, which is **not** a https://github.com/bitcoinbook/bitcoinbook/blob/173974f69e263c7de536a334224d642e6dca7d71/ch05.asciidoc#HDWalletFromSeed[master private key^]. 19 | The `CMasterKey` is encrypted by the user's passphrase. 20 | 21 | When the user changes their passphrase, they are only changing the encryption applied to the `CMasterKey`, the inner `vchCryptedKey` is not changed. 22 | This means that we do not have to read all items in the wallet database, decrypt them with the old key, encrypt them with the new key, and then write them, back to the database again. 23 | Instead, we only have to change the encryption applied to the `CMasterKey`, which is much less error-prone, and more secure. 24 | 25 | Each `CWallet` has a map of ``CMasterKey``s and when unlock is called it will try each one to see if it can decrypt and then unlock the wallet. 26 | 27 | ==== Encrypting the wallet 28 | 29 | Only private keys are encrypted. 30 | This allows us to watch for new transactions _without_ having to decrypt the wallet as each new block|transaction arrives. 31 | 32 | Decrypting the Bitcoin Core wallet requires the user to enter their passphrase, so is not convenient to do at every new block. 33 | 34 | When encrypting a wallet, a `CMasterKey` encryption key is generated, which is then sent to the `ScriptPubKeyMan` to encrypt using its `.Encrypt()` method. 35 | 36 | Once the wallet is encrypted for the first time, we re-generate all of our keys. 37 | This is to avoid the wallet using things which were not "born encrypted" in the future. 38 | For `LegacyScriptPubKeyMan` this means creating a new HD seed, and for `DescriptorScriptPubKeyMan` 8 new descriptors. 39 | 40 | If the wallet has already been used before -- while it existed in un-encrypted state -- the old ``ScriptPubKeyMan``'s are retained and so remain usable, but are not marked as `active`. 41 | The wallet will switch to the new SPKM after encryption has completed by marking the new SPKM as `active`. 42 | 43 | We take extra care during the encryption phase to either complete atomically or fail. 44 | This includes database writes where we don't want to write half and crash, for example. 45 | Therefore we will throw an assertion if the write fails. 46 | 47 | [CAUTION] 48 | ==== 49 | When you instruct a BDB database to delete a record, they are actually kept but "marked as" deleted, and _might_ be fully deleted some time in the future. 50 | 51 | This is not appropriate for our use case, for example when asking the DB to delete private keys after the wallet is encrypted for the first time. 52 | Therefore we use some https://github.com/bitcoin/bitcoin/blob/v23.0/src/wallet/wallet.cpp#L758-L765[hacks^] so that when we request deletion of unencrypted private keys from the DB, they are properly deleted immediately and not "marked as" deleted. 53 | ==== 54 | 55 | [IMPORTANT] 56 | ==== 57 | When encryption is enabled secret data must only ever exist in decrypted form in memory. 58 | ==== 59 | 60 | [CAUTION] 61 | ==== 62 | See https://github.com/bitcoin/bitcoin/pull/27080[#27080] for details on how the master key was not always cleared fully from memory after the wallet was locked. 63 | ==== 64 | -------------------------------------------------------------------------------- /wallet-exercises.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Wallet exercises 2 | :page-nav_order: 160 3 | :page-parent: Wallet 4 | == Exercises 5 | 6 | include::exercise-intro.adoc[] 7 | 8 | . Modify a wallet RPC 9 | - [ ] Create a descriptor wallet 10 | - [ ] Generate coins to yourself 11 | - [ ] Remove the `"dummy"` parameter from the `getbalance` Wallet RPC 12 | - [ ] Ensure that the `rpc_help.py` functional test passes (but ignore other test failures), fixing any errors 13 | + 14 | TIP: run `test/functional/rpc_help.py` to just run a single test 15 | - [ ] Check that the rpc call `getbalance 3 true true` passes with the `dummy` parameter removed 16 | //// 17 | - [ ] Why do we need to keep this parameter for backwards compatibility? 18 | //// 19 | 20 | . `IsMine` 21 | - [ ] Create a descriptor wallet 22 | - [ ] Generate coins to yourself 23 | - [ ] Send coins to yourself in a transaction and generate a block to confirm 24 | - [ ] Modify the wallet's `IsMine()` logic to always return `false` 25 | - [ ] Generate a new block and try to send coins to yourself in a transaction again 26 | - Observe the changes 27 | 28 | . Coin Selection 29 | - [ ] Create a descriptor wallet 30 | - [ ] Generate 200 blocks to yourself 31 | - [ ] Call `listunspent` and then `send` a large amount (e.g. 600 BTC) to yourself and observe how many inputs were used 32 | - [ ] Add a new [.underline]#preferred# coin selection algorithm to the wallet that uses *all* UTXOs in the wallet and optionally remove the other algorithms. 33 | - [ ] Redo the send and confirm that this time it will select all inputs in the wallet for the transaction 34 | 35 | . Adding a new RPC 36 | - [ ] Add a new RPC which when called will simply return to the user a random UTXO from the wallet in the form 37 | + 38 | [source,json] 39 | ---- 40 | { 41 | "txid": , 42 | "vout": 43 | } 44 | ---- 45 | 46 | //// 47 | 48 | . Wallet Database 49 | - [ ] Ensure Bitcoin Core is compiled with `sqlite` support (check you have `sqlite3` on your $PATH: `command -v sqlite3` and that your `config.status` contains an sqlite3 line: `grep sqlite3 config.status`) 50 | 51 | 52 | 53 | 54 | 55 | 56 | == Validation interface 57 | 58 | TODO 59 | 60 | == COutput 61 | 62 | TODO 63 | 64 | == HWI 65 | 66 | == Relation to consensus soft forks 67 | 68 | Much of the meat of the recently soft-forked changes (e.g. Taproot) reside not inside consensus code, but rather require improvements to the wallet. 69 | 70 | == Removed text 71 | 72 | * When adding new wallet features which will be included in the GUI, it can be good practice to first implement them as RPC commands because it's easier to create good test coverage for them. 73 | * Advanced transaction signature operations (e.g. signature aggregation, sighash flags) happen in the wallet code. 74 | 75 | === Concepts 76 | 77 | * Wallet architecture 78 | * key management 79 | ** HD wallets 80 | ** Output script descriptors 81 | * Separation of wallet and node functionality 82 | * Key Management 83 | * Transaction Construction 84 | ** Taproot 85 | ** SegWit 86 | ** Bech32 87 | ** PSBT 88 | ** Coin selection 89 | ** CPFP 90 | ** RBF 91 | ** Transaction batching 92 | ** Adaptor signatures 93 | * Multiwallet 94 | * Hardware wallet interface (HWI) 95 | * QT 96 | //// 97 | -------------------------------------------------------------------------------- /wallet-interfaces.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Wallet interfaces 2 | :page-nav_order: 60 3 | :page-parent: Wallet 4 | === Wallet interfaces 5 | 6 | include::links-onepage.adoc[] 7 | 8 | In order to facilitate code separation, distinct interfaces between the node and the wallet have been created: 9 | 10 | * The node holds a https://github.com/bitcoin/bitcoin/blob/v23.0/src/wallet/interfaces.cpp#L114[`WalletImpl`^] interface to call functions on the wallet. 11 | * The wallet holds a https://github.com/bitcoin/bitcoin/blob/v23.0/src/node/interfaces.cpp#L452[`ChainImpl`^] interface to call functions on the node. 12 | * The node notifies the wallet about new transactions and blocks through the https://github.com/bitcoin/bitcoin/blob/v23.0/src/node/interfaces.cpp#L364[`CValidationInterface`^]. 13 | 14 | TIP: For more information on `*Impl` classes see xref:appendix.adoc#pimpl-technique[PIMPL technique] in the appendix. 15 | -------------------------------------------------------------------------------- /wallet-keys.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Keys in the wallet 2 | :page-nav_order: 120 3 | :page-parent: Wallet 4 | === Keys in the wallet 5 | 6 | ==== Legacy wallet keys 7 | 8 | Legacy wallets used the "keypool" model which stored a bunch of keys. 9 | See https://github.com/bitcoin/bitcoin/blob/4b5659c6b115315c9fd2902b4edd4b960a5e066e/src/wallet/scriptpubkeyman.h#L52-L100[_src/wallet/scriptbpubkeyman.h_#L52-L100^] for historical context on the "keypool" model. 10 | 11 | The wallet would then simply iterate over each public key and generate a scriptPubKey (a.k.a. PubKey script) and address for each type of script the wallet supported. 12 | However this approach has a number of shortcomings (from least to most important): 13 | 14 | . One key could have multiple addresses 15 | . It was difficult to sign for multisig 16 | . Adding new script functionality required adding new hardcoded script types into the wallet code _for each new type of script_. 17 | 18 | Such an approach was not scalable in the long term and so a new format of wallet needed to be introduced. 19 | 20 | ==== Descriptor wallet keys 21 | 22 | Descriptor wallets instead store output script "descriptors". 23 | These descriptors can be of *any* valid script type, including arbitrary scripts which might be "unknown" to the wallet software, and this means that wallets can deterministically generate addresses for any type of valid descriptor provided by the user. 24 | 25 | Descriptors not only contain what is needed to generate an address, they also include all the script template data needed to "solve" (i.e. spend) outputs received at them. 26 | In other words they permit a valid `scriptSig` (`redeemScript` or `witnessScript`) to be generated. 27 | The document https://github.com/bitcoin/bitcoin/blob/v23.0/doc/descriptors.md[Support for Output Descriptors in Bitcoin Core^] provides more details and examples of these output descriptors. 28 | 29 | -------------------------------------------------------------------------------- /wallet-locks.adoc: -------------------------------------------------------------------------------- 1 | :page-title: Wallet locks 2 | :page-nav_order: 90 3 | :page-parent: Wallet 4 | == Wallet Locks 5 | 6 | Grepping the _src/wallet_ directory for locks, conventionally of the form `cs_*`, yields ~500 matches. 7 | For comparison the entire remainder of the codebase excluding _src/wallet/*_ yields almost 1000 matches. 8 | Many of these matches are asserts and declarations, however this still illustrates that the wallet code is highly reliant on locks to perform atomic operations with respect to the current chain state. 9 | 10 | === The `cs_wallet` lock 11 | 12 | In order to not block the rest of the program during wallet operations, each `CWallet` has its own recursive mutex `cs_wallet`: 13 | 14 | NOTE: There is currently an https://github.com/bitcoin/bitcoin/issues/19303[issue^] tracking replacement of Recursive Mutexes with Mutexes, to make locking logic easier to follow in the codebase. 15 | 16 | .src/wallet/wallet.h 17 | [source,cpp,options=nowrap] 18 | ---- 19 | /* 20 | * Main wallet lock. 21 | * This lock protects all the fields added by CWallet. 22 | */ 23 | mutable RecursiveMutex cs_wallet; 24 | ---- 25 | 26 | Most wallet operations whether reading or writing data require the use of the lock so that atomicity can be guaranteed. 27 | Some examples of wallet operations requiring the lock include: 28 | 29 | . Creating transactions 30 | . Signing transactions 31 | . Broadcasting/committing transactions 32 | . Abandoning transactions 33 | . Bumping transaction (fees) 34 | . Checking `IsMine` 35 | . Creating new addresses 36 | . Calculating balances 37 | . Creating new wallets 38 | . Importing new {priv|pub}keys/addresses 39 | . Importing/dumping wallets 40 | 41 | In addition to these higher level functions, most of ``CWallet``'s private member functions also require a hold on `cs_wallet`. 42 | 43 | === Other wallet locks 44 | 45 | . _src/wallet/bdb.cpp_, which is responsible for managing BDB wallet databases on disk, has its own mutex `cs_db`. 46 | . If external signers have been enabled (via `./configure --enable-external-signer`) then they too have their own mutex `cs_desc_man` which is acquired when descriptors are being setup. 47 | . `BlockUntilSyncedToCurrentChain()` has a unique lock exclude placed on it to prevent the caller from holding `cs_main` during its execution, and therefore prevent a possible deadlock: 48 | + 49 | .src/wallet/wallet.h 50 | [source,cpp,options=nowrap] 51 | ---- 52 | /** 53 | * Blocks until the wallet state is up-to-date to /at least/ the current 54 | * chain at the time this function is entered 55 | * Obviously holding cs_main/cs_wallet when going into this call may cause 56 | * deadlock 57 | */ 58 | void BlockUntilSyncedToCurrentChain() const LOCKS_EXCLUDED(::cs_main) EXCLUSIVE_LOCKS_REQUIRED(!cs_wallet); 59 | ---- 60 | 61 | -------------------------------------------------------------------------------- /wallet.adoc: -------------------------------------------------------------------------------- 1 | = Wallet 2 | :page-nav_order: 30 3 | :page-has_children: true 4 | 5 | TIP: This section has been updated to Bitcoin Core @ https://github.com/bitcoin/bitcoin/tree/v23.0[v23.0^] 6 | 7 | Bitcoin Core includes an optional wallet component. 8 | The wallet allows users to make and receive transactions using their own node, so that they can validate incoming payment against their own node. 9 | 10 | The wallet component has the following general aims: 11 | 12 | . Have best-in-class security 13 | ** Be extremely well tested 14 | ** Be reviewed by competent developers 15 | . Have good privacy by default 16 | . Be smart about coin selection with respect to: 17 | ** Transaction fees 18 | ** Privacy 19 | . Implement state-of-the-art features: 20 | ** Taproot 21 | ** Wallet descriptors 22 | ** Miniscript 23 | . Be backwards compatible with old (Bitcoin Core) wallet files where possible 24 | 25 | Wallets can be one of two types, "legacy" or https://github.com/bitcoin/bitcoin/blob/v23.0/doc/descriptors.md["descriptor"^]. 26 | 27 | [TIP] 28 | ==== 29 | Bitcoin Core moved to descriptor wallets as they are unambiguous as to which public keys and scripts should be used. 30 | 31 | They also simplify backups and make im/ex-porting wallet keys into other software less error-prone. 32 | ==== 33 | 34 | == Wallet overview 35 | 36 | //// 37 | * https://github.com/chaincodelabs/bitcoin-core-onboarding/tree/main/1.1_regions.asciidoc#wallet_region[Bitcoin core onboarding - wallet/^] describes the main functions of a wallet, along with some differences between legacy and descriptor wallets. 38 | //// 39 | 40 | **** 41 | Blockchain Commons provides some examples of https://github.com/BlockchainCommons/Learning-Bitcoin-from-the-Command-Line/blob/master/03_3_Setting_Up_Your_Wallet.md[Setting up a wallet] using the `bitcoin-cli tool. 42 | **** 43 | 44 | --------------------------------------------------------------------------------