├── .coveragerc ├── .editorconfig ├── .gitchangelog.rc ├── .gitchangelogrst.tpl ├── .gitignore ├── .gitlab-ci.yml ├── .readthedocs.yml ├── .travis.yml ├── AUTHORS.md ├── CHANGELOG.rst ├── DEPLOY.rst ├── INSTALL.rst ├── LICENSE.md ├── MANIFEST.in ├── README.md ├── docs ├── Makefile ├── make.bat └── source │ ├── AUTHORS.md │ ├── CHANGELOG.rst │ ├── DEPLOY.rst │ ├── INSTALL.rst │ ├── README.md │ ├── activity_aggr_file.puml │ ├── activity_aggr_lines.puml │ ├── activity_all.puml │ ├── activity_measure.puml │ ├── activity_scaling_as_torflow.puml │ ├── activity_second_relay.puml │ ├── activity_torflow_aggr.puml │ ├── activity_torflow_scaling.puml │ ├── activity_torflow_scaling_simplified.puml │ ├── activity_torflow_scaling_simplified1.puml │ ├── bandwidth_authorities.rst │ ├── bandwidth_distribution.rst │ ├── classes_original.puml │ ├── code_design.rst │ ├── conf.py │ ├── config.default.ini │ ├── config.example.ini │ ├── config.log.default.ini │ ├── config.rst │ ├── config_tor.rst │ ├── contributing.rst │ ├── data │ ├── critical_sections.dia │ ├── scanner.dia │ └── use_cases_data_sources.dia │ ├── differences.rst │ ├── documenting.rst │ ├── examples │ └── sbws.example.ini │ ├── faq.rst │ ├── generator.rst │ ├── glossary.rst │ ├── how_works.rst │ ├── images │ ├── 20180901_163442.png │ ├── 20180901_164014.png │ ├── 20210111_consensushealth_bwauths.png │ ├── 43710932-ac1eeea8-9960-11e8-9e7e-21fddff2f7a3.png │ ├── 43710933-ac95e0bc-9960-11e8-9aaf-0bb1f83b65e2.png │ ├── activity_aggr_file.svg │ ├── activity_aggr_lines.svg │ ├── activity_all.svg │ ├── activity_measure.svg │ ├── activity_scaling_as_torflow.svg │ ├── activity_second_relay.svg │ ├── activity_torflow_aggr.svg │ ├── activity_torflow_scaling.svg │ ├── activity_torflow_scaling_simplified.svg │ ├── activity_torflow_scaling_simplified1.svg │ ├── advertised_bandwidth.png │ ├── bwauth.svg │ ├── bwauth_measured_7days.png │ ├── bwauth_measured_90days.png │ ├── classes_original.svg │ ├── critical_sections.svg │ ├── dirauths_bwauths.png │ ├── packages_sbws.svg │ ├── pycallgraph.png │ ├── scanner.svg │ ├── threads.svg │ ├── torperf.png │ ├── totalcw.png │ ├── use_cases_classes.svg │ └── use_cases_data_sources.svg │ ├── implementation.rst │ ├── index.rst │ ├── man_sbws.ini.rst │ ├── man_sbws.rst │ ├── monitoring_bandwidth.rst │ ├── proposals │ └── 001-switchtohttp.rst │ ├── roadmap.rst │ ├── sbws.core.rst │ ├── sbws.lib.rst │ ├── sbws.rst │ ├── sbws.util.rst │ ├── state.rst │ ├── testing.rst │ ├── threads.puml │ ├── tor_bandwidth_files.rst │ ├── torflow_aggr.rst │ └── v3bw.txt ├── sbws ├── __init__.py ├── _version.py ├── config.default.ini ├── config.log.default.ini ├── core │ ├── __init__.py │ ├── bwfile_health.py │ ├── cleanup.py │ ├── generate.py │ ├── scanner.py │ └── stats.py ├── globals.py ├── lib │ ├── __init__.py │ ├── bwfile_health.py │ ├── circuitbuilder.py │ ├── destination.py │ ├── heartbeat.py │ ├── relaylist.py │ ├── relayprioritizer.py │ ├── resultdump.py │ ├── scaling.py │ └── v3bwfile.py ├── sbws.py └── util │ ├── __init__.py │ ├── config.py │ ├── filelock.py │ ├── fs.py │ ├── iso3166.py │ ├── json.py │ ├── parser.py │ ├── requests.py │ ├── state.py │ ├── stem.py │ ├── timestamp.py │ ├── timestamps.py │ └── userquery.py ├── scripts ├── maint │ ├── release.py │ ├── update-authors │ └── update-website └── tools │ ├── get-per-relay-budget.py │ ├── osx-extra-loopback.sh │ ├── sbws-http-server.py │ └── scale-v3bw-with-budget.py ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── conftest.py ├── data │ ├── .sbws │ │ ├── datadir │ │ │ └── 2019-03-25.txt │ │ ├── state.dat │ │ ├── tor │ │ │ ├── cached-consensus │ │ │ └── cached-descriptors │ │ └── v3bw │ │ │ └── 20190325_130909.v3bw │ ├── 2020-02-29-10-00-00-consensus │ ├── 2020-02-29-10-05-00-server-descriptors │ ├── 2020-02-29-11-00-00-consensus │ ├── 2020-03-05-10-00-00-consensus │ └── 2020-03-22-08-35-00-bandwidth ├── integration │ ├── __init__.py │ ├── chutney_data │ │ ├── bwscanner │ │ ├── client_bwscanner.tmpl │ │ ├── non-exit.tmpl │ │ ├── relay-MAB.tmpl │ │ └── relay-MBR.tmpl │ ├── conftest.py │ ├── core │ │ └── test_scanner.py │ ├── lib │ │ ├── test_circuitbuilder.py │ │ ├── test_destination.py │ │ ├── test_relaylist.py │ │ └── test_relayprioritizer.py │ ├── run.sh │ ├── sbws_testnet.ini │ ├── start_chutney.sh │ ├── stop_chutney.sh │ ├── test_files.py │ └── util │ │ ├── __init__.py │ │ ├── test_requests.py │ │ └── test_stem.py └── unit │ ├── __init__.py │ ├── conftest.py │ ├── core │ ├── test_generate.py │ ├── test_scanner.py │ └── test_stats.py │ ├── globals.py │ ├── lib │ ├── data │ │ ├── results.txt │ │ ├── results_0_consensus_bw.txt │ │ ├── results_away.txt │ │ ├── results_no_consensus_bw.txt │ │ ├── results_no_desc_bw_avg.txt │ │ ├── results_no_desc_bw_avg_obs.txt │ │ └── results_no_desc_bw_obs.txt │ ├── test_destination.py │ ├── test_heartbeat.py │ ├── test_relaylist.py │ ├── test_relayprioritizer.py │ ├── test_resultdump.py │ ├── test_results.py │ ├── test_scaling.py │ └── test_v3bwfile.py │ ├── test_bwfile_health.py │ └── util │ ├── data │ └── user_sbws.ini │ ├── test_config.py │ ├── test_json.py │ ├── test_state.py │ ├── test_stem.py │ ├── test_timestamp.py │ ├── test_timestamps.py │ └── test_userquery.py ├── tox.ini └── versioneer.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = */__init__.py 3 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # this should work for all editors that support .editorconfig! 2 | # 3 | # on debian, emacs users should install elpa-editorconfig and vim 4 | # users should install vim-editorconfig. 5 | 6 | root = true 7 | 8 | [*] 9 | indent_style = space 10 | # this remove EOF new line with some editors 11 | # insert_final_newline = true 12 | trim_trailing_whitespace = true 13 | end_of_line = lf 14 | charset = utf-8 15 | max_line_length = 79 16 | 17 | [*.py] 18 | indent_size = 4 19 | 20 | [Makefile] 21 | indent_style = tab 22 | 23 | [*.ini] 24 | indent_size = 4 25 | 26 | [*.yml] 27 | indent_size = 2 28 | -------------------------------------------------------------------------------- /.gitchangelogrst.tpl: -------------------------------------------------------------------------------- 1 | % if data["title"]: 2 | ${data["title"]} 3 | \n 4 | ${"=" * len(data["title"])} 5 | 6 | 7 | % endif 8 | % for version in data["versions"]: 9 | <% 10 | title = "%s (%s)" % (version["tag"], version["date"]) if version["tag"] else opts["unreleased_version_label"] 11 | 12 | nb_sections = len(version["sections"]) 13 | %>${title} 14 | ${"-" * len(title)} 15 | % for section in version["sections"]: 16 | % if not (section["label"] == "Other" and nb_sections == 1): 17 | 18 | ${section["label"]} 19 | ${"~" * len(section["label"])} 20 | % endif 21 | % for commit in section["commits"]: 22 | <% 23 | subject = "%s [%s]" % (commit["subject"], ", ".join(commit["authors"])) 24 | entry = indent('\n'.join(textwrap.wrap(subject)), 25 | first="- ").strip() 26 | %>${entry} 27 | 28 | % if commit["body"]: 29 | ${indent(commit["body"])} 30 | % endif 31 | % endfor 32 | % endfor 33 | 34 | % endfor 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | __pycache__/ 3 | venv*/ 4 | passwords.txt 5 | *.swp 6 | *.egg-info/ 7 | docs/build 8 | .tox 9 | .coverage 10 | htmlcov 11 | .pytest_cache 12 | dist 13 | build 14 | *.lockfile 15 | chutney 16 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | # core/tor releases: 2 | # https://gitlab.torproject.org/tpo/core/team/-/wikis/NetworkTeam/CoreTorReleases 3 | # As of 2021.03.02 all dirauths are running with version greater or equal to 4 | # 0.4.5. 5 | # 0.4.6 stable by Jun 15, 2021 6 | # 0.4.5 (LTS) EOL Feb 15, 2023 7 | # 0.3.5 (LTS) EOL Feb 1, 2022 8 | # Python releases: 9 | # 3.10 stable by 2021-10-04: https://www.python.org/dev/peps/pep-0619/ 10 | # Python stable releases: https://www.python.org/downloads/ 11 | # 3.9 EOL 2025-10 PEP 596 12 | # 3.8 EOL 2024-10 PEP 569, newest major release 13 | # 3.7 EOL 2023-06-27 PEP 537, included in Debian buster 14 | # 3.6 EOL 2021-12-23 PEP 494 15 | 16 | variables: 17 | BASE_IMAGE: python:3.8 18 | RELEASE: tor-nightly-master-buster 19 | # Without version, the default available in the Debian repository will be 20 | # installed. 21 | # Specifying which version starts with will install the highest that start 22 | # with that version. 23 | TOR: tor=* 24 | PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" 25 | 26 | cache: 27 | paths: 28 | - .cache/pip 29 | 30 | image: $BASE_IMAGE 31 | 32 | before_script: 33 | - "wget https://deb.torproject.org/torproject.org/\ 34 | A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc" 35 | - cat A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc | apt-key add - 36 | - echo deb [signed-by=A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89] 37 | http://deb.torproject.org/torproject.org $RELEASE 38 | main >> /etc/apt/sources.list 39 | - apt update -yqq 40 | - apt install -yqq $TOR 41 | - pip install tox 42 | - python --version 43 | - tor --version 44 | 45 | after_script: 46 | - tox -e stats 47 | 48 | python36: 49 | variables: 50 | BASE_IMAGE: python:3.6 51 | image: $BASE_IMAGE 52 | script: 53 | - tox -e py36 54 | - tox -e integration 55 | 56 | python37tor035: 57 | variables: 58 | BASE_IMAGE: python:3.7 59 | RELEASE: tor-nightly-0.3.5.x-buster 60 | TOR: tor=0.3.5* 61 | image: $BASE_IMAGE 62 | script: 63 | - tox -e py37 64 | - tox -e integration 65 | 66 | python37tor045: 67 | variables: 68 | BASE_IMAGE: python:3.7 69 | RELEASE: tor-nightly-0.4.5.x-buster 70 | TOR: tor=0.4.5* 71 | image: $BASE_IMAGE 72 | script: 73 | - tox -e py37 74 | - tox -e integration 75 | 76 | python37tormaster: 77 | variables: 78 | BASE_IMAGE: python:3.7 79 | RELEASE: tor-nightly-master-buster 80 | TOR: tor=0.4.6* 81 | image: $BASE_IMAGE 82 | script: 83 | - tox -e py37 84 | - tox -e integration 85 | 86 | python37torstable: 87 | variables: 88 | BASE_IMAGE: python:3.7 89 | RELEASE: buster 90 | TOR: tor 91 | image: $BASE_IMAGE 92 | script: 93 | - tox -e py37 94 | - tox -e integration 95 | 96 | python38: 97 | # This will overwrite the default before_script, so need to repeat the 98 | # commands 99 | before_script: 100 | - "wget https://deb.torproject.org/torproject.org/\ 101 | A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc" 102 | - cat A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc | apt-key add - 103 | - echo deb [signed-by=A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89] 104 | http://deb.torproject.org/torproject.org $RELEASE 105 | main >> /etc/apt/sources.list 106 | - apt update -yqq 107 | - apt install -yqq $TOR 108 | - pip install tox 109 | - python --version 110 | - tor --version 111 | # To build the docs 112 | - apt install -yqq texlive-latex-extra 113 | - apt install -yqq dvipng 114 | image: $BASE_IMAGE 115 | script: 116 | - tox -e inst 117 | - tox -e setup 118 | - tox -e py38 119 | - tox -e integration 120 | - tox -e lint 121 | - tox -e doc 122 | 123 | python39: 124 | variables: 125 | BASE_IMAGE: python:3.9 126 | image: $BASE_IMAGE 127 | script: 128 | - tox -e py39 129 | - tox -e integration 130 | 131 | python310: 132 | variables: 133 | BASE_IMAGE: python:3.10-rc-buster 134 | image: $BASE_IMAGE 135 | script: 136 | - tox -e py310 137 | - tox -e integration 138 | allow_failure: true 139 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yml 2 | 3 | build: 4 | image: latest 5 | 6 | python: 7 | version: 3.5 8 | # To run "pip install ." in rtfd.io 9 | pip_install: true 10 | 11 | # To run "pip install .[docs]" in rtfd.io 12 | extra_requirements: 13 | - docs 14 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | # The default python version on Travis bionic is 3.6 3 | # But we add this line to show the python version in the Travis UI 4 | python: "3.6" 5 | 6 | sudo: false 7 | 8 | cache: pip 9 | 10 | env: 11 | matrix: 12 | ## This matrix entry is required, but it doesn't actually create any jobs 13 | ## by itself. All jobs are created by matrix: include: entries 14 | ## 15 | ## The TOR env var should be kept in sync with the Linux tor version in 16 | ## the addons section below 17 | - TOR="master-nightly" 18 | 19 | matrix: 20 | # include creates Linux, python 3.6, tor master builds by default 21 | # we use tor master to catch tor issues before stable releases 22 | # the key(s) in each item override these defaults 23 | include: 24 | ## Test all supported and available tor versions on Linux 25 | ## If the deb.torproject.org repositories are removed, we will fall back to 26 | ## Ubuntu security's tor version (currently 0.2.9.14). We might want to 27 | ## automatically fail the job if we can't get a newer tor, see #29741. 28 | ## 29 | ## The current tor versions in Ubuntu are on this page: 30 | ## https://packages.ubuntu.com/search?keywords=tor&searchon=names&exact=1 31 | ## 32 | 33 | - addons: 34 | apt: 35 | sources: 36 | - sourceline: 'deb https://deb.torproject.org/torproject.org tor-nightly-0.3.5.x-bionic main' 37 | key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc' 38 | packages: 39 | - tor 40 | env: TOR="0.3.5-nightly" 41 | - addons: 42 | apt: 43 | sources: 44 | - sourceline: 'deb https://deb.torproject.org/torproject.org tor-nightly-0.4.1.x-bionic main' 45 | key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc' 46 | packages: 47 | - tor 48 | env: TOR="0.4.1-nightly" 49 | ## The current stable release is 0.4.2 50 | - addons: 51 | apt: 52 | sources: 53 | - sourceline: 'deb https://deb.torproject.org/torproject.org bionic main' 54 | key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc' 55 | packages: 56 | - tor 57 | env: TOR="stable-release" 58 | - addons: 59 | apt: 60 | sources: 61 | - sourceline: 'deb https://deb.torproject.org/torproject.org tor-nightly-0.4.2.x-bionic main' 62 | key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc' 63 | packages: 64 | - tor 65 | env: TOR="0.4.2-nightly" 66 | # We test nightly master by default, in our network tests. So we don't need 67 | # a nightly master test here. 68 | 69 | ## Test all supported python releases 70 | 71 | ## Pre-installed in Travis Bionic: 72 | ## https://docs.travis-ci.com/user/reference/bionic/#python-support 73 | 74 | ## End of Life: December 2021 75 | ## https://www.python.org/dev/peps/pep-0494/#lifespan 76 | - python: "3.6" 77 | 78 | ## End of Life: June 2023 79 | ## https://www.python.org/dev/peps/pep-0537/#lifespan 80 | - python: "3.7" 81 | 82 | ## Extra Installs 83 | 84 | ## End of Life: October 2024 85 | ## https://www.python.org/dev/peps/pep-0569/#lifespan 86 | - python: "3.8" 87 | 88 | ## Python 3.9 89 | ## (Add 3.9-dev), so far is the same as nightly 90 | ## Stable: 10 October 2020 91 | ## (Switch from 3.9-dev to 3.9, and check for {3.10,4.0}-dev) 92 | ## End of Life: October 2025 93 | ## https://www.python.org/dev/peps/pep-0596/#lifespan 94 | 95 | - python: "nightly" 96 | 97 | allow_failures: 98 | # stem fails: 99 | # Error initting controller socket: module 'collections' has no 100 | # attribute 'Iterable' 101 | - python: nightly 102 | 103 | ## (Linux only) Use the Ubuntu Bionic Linux Image 104 | dist: bionic 105 | 106 | ## Download our dependencies 107 | addons: 108 | ## (Linux only) 109 | apt: 110 | sources: 111 | - sourceline: 'deb https://deb.torproject.org/torproject.org tor-nightly-master-bionic main' 112 | key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc' 113 | packages: 114 | - tor 115 | # To build the docs 116 | - texlive-latex-extra 117 | - dvipng 118 | 119 | install: 120 | - pip install tox-travis 121 | - dpkg-query --show 122 | ## Use the default spelling for python, unless it is overridden 123 | - export PYTHON=${PYTHON:-python} 124 | - $PYTHON --version 125 | - tor --version 126 | - tox --version 127 | 128 | script: 129 | - tox 130 | # This is not in included in the tox envlist, in order to don't need Internet 131 | # when running tox 132 | # - tox -e doclinks 133 | - tox -e clean 134 | 135 | after_success: 136 | # gather Python coverage 137 | - tox -e stats 138 | 139 | notifications: 140 | irc: 141 | channels: 142 | - "irc.oftc.net#tor-ci" 143 | template: 144 | - "%{repository_slug} %{branch} %{commit} - %{author}: %{commit_subject}" 145 | - "Build #%{build_number} %{result}. Details: %{build_url}" 146 | -------------------------------------------------------------------------------- /AUTHORS.md: -------------------------------------------------------------------------------- 1 | Authors 2 | ======= 3 | 4 | The following people have contributed to Simple Bandwidth Scanner. 5 | Thank you for helping make Tor better. 6 | 7 | * anadahz 8 | * George Kadianakis 9 | * Georg Koppen 10 | * juga 11 | * Matt Traudt 12 | * teor 13 | 14 | *Last updated: 2020-06-26 on d7a822bf* 15 | -------------------------------------------------------------------------------- /DEPLOY.rst: -------------------------------------------------------------------------------- 1 | .. _deploy: 2 | 3 | Deploying Simple Bandwidth Scanner 4 | ===================================== 5 | 6 | To run sbws is needed: 7 | 8 | - A machine to run the :term:`scanner`. 9 | - One or more :term:`destination` (s) that serve a large file. 10 | 11 | Both the ``scanner`` and your the ``destination`` (s) should be on fast, 12 | well connected machines. 13 | 14 | .. _destinations_requirements: 15 | 16 | destination requirements 17 | ------------------------------------ 18 | 19 | - A Web server installed and running that supports HTTP GET, HEAD and 20 | Range (:rfc:`7233`) requests. 21 | ``Apache`` HTTP Server and ``Nginx`` support them. 22 | - TLS support to avoid HTTP content caches at the various exit nodes. 23 | - Certificates can be self-signed. 24 | - A large file; at the time of writing, at least 1 GiB in size 25 | It can be created running:: 26 | 27 | head -c $((1024*1024*1024)) /dev/urandom > 1GiB 28 | 29 | - A fixed IP address or a domain name. 30 | - Bandwidth: at least 12.5MB/s (100 Mbit/s). 31 | - Network traffic: around 12-15GB/day. 32 | 33 | If possible, use a `Content delivery network`_ (CDN) in order to make the 34 | destination IP closer to the scanner exit. 35 | 36 | scanner setup 37 | ---------------------- 38 | 39 | Install sbws according to ``_ (in the local directory or Tor 40 | Project Gitlab) or ``_ (local build or Read the Docs). 41 | 42 | To run the ``scanner`` it is mandatory to create a configuration file with at 43 | least one ``destination``. 44 | It is recommended to set several ``destinations`` so that the ``scanner`` can 45 | continue if one fails. 46 | 47 | If ``sbws`` is installed from the Debian package, then create the configuration 48 | file in ``/etc/sbws/sbws.ini``. 49 | You can see an example with all the possible options here, note that you don't 50 | need to include all of that and that everything that starts with ``#`` and 51 | ``;`` is a comment: 52 | 53 | .. literalinclude:: /examples/sbws.example.ini 54 | :caption: Example sbws.example.ini 55 | 56 | If ``sbws`` is installed from the sources as a non-root user then create the 57 | configuration file in ``~/.sbws.ini``. 58 | 59 | More details about the configuration file can be found in 60 | ``./docs/source/man_sbws.ini.rst`` (in the local directory or Tor Project 61 | Gitlab) or ``_ (local build or Read the Docs) or 62 | ``man sbws.ini`` (system package). 63 | 64 | See also ``./docs/source/man_sbws.rst`` (in the local directory or Tor Project 65 | Gitlab) or ``_ (local build or Read the Docs) or ``man sbws`` 66 | (system package). 67 | 68 | .. _Content delivery network: https://en.wikipedia.org/wiki/Content_delivery_network 69 | -------------------------------------------------------------------------------- /INSTALL.rst: -------------------------------------------------------------------------------- 1 | .. _install: 2 | 3 | Installing Simple Bandwidth Scanner 4 | =================================== 5 | 6 | The recommended method is to install it from your system package manager. 7 | 8 | In Debian_/Ubuntu_ systems:: 9 | 10 | sudo apt install sbws 11 | 12 | To install also the documentation:: 13 | 14 | sudo apt install sbws-doc 15 | 16 | You might need to check in which releases is the package available. 17 | 18 | There is a port_ for FreeBSD. 19 | 20 | Continue reading to install ``sbws`` in other ways. 21 | 22 | System requirements 23 | -------------------- 24 | 25 | - Tor (last stable version is recommended) 26 | - Python 3 (>= 3.6) 27 | 28 | Python dependencies 29 | -------------------- 30 | 31 | - Stem_ >= 1.7.0 32 | - Requests_ (with socks_ support) >= 2.10.0 33 | 34 | It is recommend to install the dependencies from your system package manager. 35 | If that is not possible, because the Python dependencies are not available in 36 | your system, you can install them from their sources. 37 | We only recommend using pip_ for development or testing. 38 | 39 | Installing sbws from source 40 | --------------------------- 41 | 42 | Clone ``sbws``:: 43 | 44 | git clone https://git.torproject.org/sbws.git 45 | git checkout maint-1.1 46 | 47 | The branch ``maint-1.1`` is the last stable version and the one that should be 48 | used in production. 49 | 50 | and install it:: 51 | 52 | cd sbws 53 | python3 setup.py install 54 | 55 | Installing sbws for development or testing 56 | ------------------------------------------ 57 | 58 | If you use pip_, it is recommended to use virtualenv_, to avoid having 59 | different versions of the same libraries in your system. 60 | 61 | To create a ``virtualenv``:: 62 | 63 | virtualenv venv -p /usr/bin/python3 64 | source venv/bin/activate 65 | 66 | Clone ``sbws``:: 67 | 68 | git clone https://git.torproject.org/sbws.git 69 | 70 | Install the python dependencies:: 71 | 72 | cd sbws && pip install -e . 73 | 74 | Configuration and deployment 75 | ---------------------------- 76 | 77 | ``sbws`` needs :term:`destination` s to request files from. 78 | 79 | Please, see ``_ (in the local directory or Tor Project Gitlab) or 80 | ``_ (local build or Read the Docs) 81 | to configure, deploy and run ``sbws``. 82 | 83 | System physical requirements 84 | ----------------------------- 85 | 86 | - Bandwidth: at least 12.5MB/s (100 Mbit/s). 87 | - Free RAM: at least 2GB 88 | - Free disk: at least 3GB 89 | 90 | ``sbws`` and its dependencies need around 20MB of disk space. 91 | After 57 days ``sbws`` data files use a maximum of 3GB. 92 | If ``sbws`` is configured to log to files (by default will log to the 93 | system log), it will need a maximum of 500MB. 94 | 95 | It is recommended to set up an automatic disk space monitoring on ``sbws`` data 96 | and log partitions. 97 | 98 | Details about ``sbws`` data: 99 | 100 | ``sbws`` produces around 100MB of data a day. 101 | By default raw results' files are compressed after 29 days and deleted after 102 | 57. 103 | The bandwidth files are compressed after 7 days and deleted after 1. 104 | After 57 days, the disk space used by the data will be up to 3GB. 105 | It will not increase further. 106 | If ``sbws`` is configured to log to files, logs will be rotated after they 107 | are 10MB and it will keep 50 rotated log files. 108 | 109 | .. _virtualenv: https://virtualenv.pypa.io/en/stable/installation/ 110 | .. _Stem: https://stem.torproject.org/ 111 | .. _socks: http://docs.python-requests.org/en/master/user/advanced/#socks 112 | .. https://readthedocs.org/projects/requests/ redirect to this, but the 113 | .. certificate of this signed by rtd 114 | .. _Requests: http://docs.python-requests.org/ 115 | .. http://flake8.pycqa.org/ certificate is signed by rtf 116 | .. _Flake8: https://flake8.readthedocs.org/ 117 | .. _pytest: https://docs.pytest.org/ 118 | .. _tox: https://tox.readthedocs.io 119 | .. _Coverage: https://coverage.readthedocs.io/ 120 | .. _port: https://www.freshports.org/net/py-sbws/ 121 | .. _Debian: https://packages.debian.org/search?keywords=sbws&searchon=names&suite=all§ion=all 122 | .. _Ubuntu: https://packages.ubuntu.com/search?keywords=sbws&searchon=names&suite=all§ion=all 123 | .. _pip: https://pypi.org/project/pip/ 124 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.md 2 | include *.rst 3 | include *.ini 4 | recursive-include docs * 5 | prune docs/build 6 | recursive-include tests * 7 | recursive-exclude **/__pycache__ * 8 | include versioneer.py 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Readme 2 | ====== 3 | 4 | [![Build Status](https://travis-ci.org/torproject/sbws.svg?branch=master)](https://travis-ci.org/https://travis-ci.org/torproject/sbws) 5 | 6 | Simple Bandwidth Scanner (called `sbws`) is a Tor bandwidth scanner that 7 | generates bandwidth files to be used by Directory Authorities. 8 | 9 | The scanner measures the bandwidth of each relay in the Tor network 10 | (except the directory authorities) by creating a two hops circuit 11 | with the relay. It then measures the bandwidth by downloading data 12 | from a destination Web Server and stores the measurements. 13 | 14 | The generator read the measurements, aggregates, filters and 15 | scales them using torflow's scaling method. 16 | Then it generates a bandwidth list file that is read 17 | by a directory authority to report relays’ bandwidth in its vote. 18 | 19 | **WARNING**: This software is intended to be run by researchers using a test 20 | Tor network, such as chutney or shadow, or by the Tor bandwidth authorities 21 | on the public Tor network. 22 | Please do not run this software on the public Tor network unless you are one 23 | of the Tor bandwidth authorities, to avoid creating unnecessary traffic. 24 | 25 | **ADVICE**: It is recommended to read this documentation at 26 | [Read the Docs](https://sbws.rtfd.io). In 27 | [Tor Project Gitlab](https://gitlab.torproject.org/tpo/network-health/sbws) 28 | (tpo Gitlab) some links won't be properly rendered. 29 | It can also be read after installing the Debian package ``sbws-doc`` in 30 | ``/usr/share/doc/sbws`` or after building it locally as explained in 31 | ``./docs/source/documenting.rst``. 32 | 33 | 34 | Installing 35 | ------------ 36 | 37 | See [./INSTALL.rst](INSTALL.rst) (in local directory or tpo Gitlab) or 38 | [INSTALL.html](INSTALL.html) (local build or Read the Docs). 39 | 40 | Deploying and running 41 | --------------------- 42 | 43 | See [./DEPLOY.rst](DEPLOY.rst) (in local directory or tpo Gitlab) or 44 | [DEPLOY.html](DEPLOY.html) (local build or Read the Docs). 45 | 46 | Changelog 47 | -------------- 48 | 49 | See [./CHANGELOG.rst](CHANGELOG.rst) (in local directory or tpo Gitlab) or 50 | [CHANGELOG.html](CHANGELOG.html) (local build or Read the Docs). 51 | 52 | Documentation 53 | -------------- 54 | 55 | More extensive documentation can be found in the ``./docs`` directory, 56 | and online at [sbws.rtfd.io](https://sbws.readthedocs.io). 57 | 58 | ## License 59 | 60 | This work is in the public domain within the United States. 61 | 62 | We waive copyright and related rights in the work worldwide through the 63 | [CC0-1.0 license](https://creativecommons.org/publicdomain/zero/1.0). 64 | 65 | You can find a copy of the CC0 Public Domain Dedication along with this 66 | software in ./LICENSE.md 67 | 68 | ## Authors 69 | 70 | See [./AUTHORS.md](AUTHORS.md) (in local directory or tpo Gitlab) or 71 | [AUTHORS.html](AUTHORS.html) (local build or Read the Docs). 72 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = simple-bw-scanner 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | # generate SVG 11 | IMAGEDIRS = $(SOURCEDIR)/images 12 | BUILDDIRIMAGES = $(BUILDDIR)/html/_images 13 | PYREVERSE = pyreverse 14 | PYREVERSE_FLAGS = -o svg -p sbws ../sbws 15 | UMLSVG := $(PYREVERSE) $(PYREVERSE_FLAGS);mv *.svg $(IMAGEDIRS);mkdir -p $(BUILDDIRIMAGES);cp $(IMAGEDIRS)/*.svg $(BUILDDIRIMAGES) 16 | PLANTUML := plantuml 17 | PLANTUML_CMD := $(PLANTUML) -tsvg -o ../$(IMAGEDIRS) $(SOURCEDIR)/*.puml 18 | 19 | # Put it first so that "make" without argument is like "make help". 20 | help: 21 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 22 | 23 | .PHONY: help Makefile 24 | 25 | umlsvg: 26 | @echo "Generating UML SVG" 27 | $(UMLSVG) 28 | 29 | plantuml: 30 | @echo "Generating plantuml .svg files." 31 | $(PLANTUML_CMD) 32 | 33 | # Catch-all target: route all unknown targets to Sphinx using the new 34 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 35 | %: Makefile 36 | # commented because if system packages invoke make html, it'll automatically 37 | # recreate the svg on every build, and it's not deterministic. 38 | #$(UMLSVG) 39 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 40 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | set SPHINXPROJ=simple-bw-scanner 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 20 | echo.installed, then set the SPHINXBUILD environment variable to point 21 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 22 | echo.may add the Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /docs/source/AUTHORS.md: -------------------------------------------------------------------------------- 1 | ../../AUTHORS.md -------------------------------------------------------------------------------- /docs/source/CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | ../../CHANGELOG.rst -------------------------------------------------------------------------------- /docs/source/DEPLOY.rst: -------------------------------------------------------------------------------- 1 | ../../DEPLOY.rst -------------------------------------------------------------------------------- /docs/source/INSTALL.rst: -------------------------------------------------------------------------------- 1 | ../../INSTALL.rst -------------------------------------------------------------------------------- /docs/source/README.md: -------------------------------------------------------------------------------- 1 | ../../README.md -------------------------------------------------------------------------------- /docs/source/activity_aggr_file.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | title Activity diagram sbws relays exclusion (V3BWFile.from_results) 4 | 5 | start 6 | :router_statuses_d; 7 | while (results.items()?) 8 | :line, reason = V3BWLine.from_results(); 9 | if (not reason?) then (yes) 10 | :bwlines_raw.append(line); 11 | else (no) 12 | :bw_lines_excluded.append(line); 13 | :exclusion_dict[reason] = exclusion_dict.get(reason, 0) + 1; 14 | endif 15 | endwhile 16 | :header.add_relays_excluded_counters(exclusion_dict); 17 | if (not bw_lines_raw?) then (yes) 18 | :return (header, bw_lines_excluded); 19 | stop 20 | endif 21 | if (scaling_method == TORFLOW?) then (yes) 22 | :bw_lines = cls.bw_torflow_scale(); 23 | endif 24 | :return (header, bw_lines + bw_lines_excluded); 25 | stop 26 | 27 | @enduml 28 | -------------------------------------------------------------------------------- /docs/source/activity_aggr_lines.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | title Activity diagram sbws results exclussion and aggregation (V3BWLine.from_results) 4 | 5 | start 6 | :success_results; 7 | if (not success_results?) then (yes) 8 | :return (, exclusion_reason); 9 | stop 10 | endif 11 | :results_away; 12 | if (not results_away?) then (yes) 13 | :return (, exclusion_reason); 14 | stop 15 | endif 16 | :results_recent; 17 | if (not results_recent?) then (yes) 18 | :return (, exclusion_reason); 19 | stop 20 | endif 21 | if (node_id in router_statuses_d?) then (yes) 22 | :consensus_bandwidth; 23 | else (no) 24 | :consensus_bandwidth; 25 | endif 26 | :obs_last; 27 | if (obs_last is None and consensus_bandwidth is None?) then (yes) 28 | :return(cls(node_id, 1), "no_consensus_no_observed_bw"); 29 | stop 30 | endif 31 | :bw; 32 | :kwargs[...]; 33 | :return (node_id, bw, **kwargs), None; 34 | stop 35 | 36 | @enduml 37 | -------------------------------------------------------------------------------- /docs/source/activity_all.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | start 4 | 5 | while (no SIGINT/SIGTERM?) 6 | 7 | while (next relay to measure?) 8 | 9 | :Select a destination; 10 | 11 | :Select a second relay; 12 | 13 | :Build a circuit; 14 | 15 | :HTTP GET (Range-Bytes); 16 | 17 | :Store measurement; 18 | 19 | endwhile 20 | 21 | endwhile 22 | 23 | stop 24 | 25 | @enduml -------------------------------------------------------------------------------- /docs/source/activity_measure.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | start 4 | 5 | if (exit?) then (yes) 6 | :[h, r]; 7 | else (no) 8 | :[r, h]; 9 | endif 10 | if (circuit?) then (yes) 11 | :stream; 12 | if (no stream and [h, r]) then (yes) 13 | :[r, h] (r is exit); 14 | if (circuit?) then (yes) 15 | :stream; 16 | else (no) 17 | :WARN; 18 | :ErrorCircuit; 19 | endif 20 | endif 21 | if (no stream) then (yes) 22 | :ErrorStream; 23 | endif 24 | else (no) 25 | :ErrorCircuit; 26 | endif 27 | 28 | stop 29 | 30 | @enduml 31 | -------------------------------------------------------------------------------- /docs/source/activity_scaling_as_torflow.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | title Activity diagram sbws bw_torflow_scale 4 | 5 | start 6 | 'bw_lines: the relays' raw measurements 7 | :mu = mean([l.bw_mean for l in bw_lines]); 8 | :muf = mean([l.bw_filt for l in bw_lines]); 9 | :sum_bw = 0; 10 | while (for l in bw_lines?) 11 | :bw_obs; 12 | :desc_bw = min(desc_bw_obs, l.desc_bw_bur, l.desc_bw_avg); 13 | :min_bandwidth = min(desc_bw, l.consensus_bandwidth); 14 | :ratio_stream = l.bw_mean / mu; 15 | :ratio_stream_filtered = l.bw_filt / muf; 16 | :ratio = max(ratio_stream, ratio_stream_filtered); 17 | :l.bw = ratio * min_bandwidth; 18 | if (router_statuses_d?) then (yes) 19 | if (l.node_id in router_statuses_d?) then (yes) 20 | :sum_bw += l.bw; 21 | endif 22 | else (no) 23 | :sum_bw += l.bw; 24 | endif 25 | endwhile 26 | :hlimit = sum_bw * cap; 27 | while (for l in bw_lines?) 28 | :bw_scaled = min(hlimit, l.bw); 29 | :l.bw = kb_round_x_sig_dig(bw_scaled, digits=num_round_dig); 30 | endwhile 31 | :return sorted(bw_lines_tf, key=lambda x: x.bw, reverse=reverse); 32 | stop 33 | 34 | footer last updated 2021-01-08 35 | @enduml 36 | -------------------------------------------------------------------------------- /docs/source/activity_second_relay.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | start 4 | 5 | if (relay to measure is exit?) then (yes) 6 | :obtain non-exits; 7 | else (no) 8 | :obtain an exits 9 | without bad flag 10 | that can exit 11 | to port 443; 12 | endif 13 | :potential second relays; 14 | :obtain a relay 15 | from potential 16 | sencond relays 17 | randomly; 18 | if (second relay has 2x bandwidth?) then (yes) 19 | elseif (other second relay has 1.5x bandwidth?) then (yes) 20 | elseif (other second relay has 1x bandwidth?) then (yes) 21 | else (nothing) 22 | stop 23 | endif 24 | :second relay selected!; 25 | :Build a circuit 26 | whith exit as 27 | second hop; 28 | stop 29 | 30 | @enduml -------------------------------------------------------------------------------- /docs/source/activity_torflow_aggr.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | title "Activity diagram Torflow measurements aggregation." 3 | 4 | ' Constants in consensus 5 | ' :Wbd=0 Wbe=0 Wbg=4148 Wbm=10000 Wdb=10000 Web=10000 Wed=10000 Wee=10000 Weg=10000 Wem=10000 Wgb=10000 Wgd=0 Wgg=5852 Wgm=5852 Wmb=10000 Wmd=0 Wme=0 Wmg=4148 Wmm=10000; 6 | ' Constants in code 7 | ' ;IGNORE_GUARD = 0 GUARD_SAMPLE_RATE = 2*7*24*60*60 # 2wks MAX_AGE = 2*GUARD_SAMPLE_RATE; 8 | ' ;K_p = 1.0 T_i =0 T_i_decay = 0 T_d = 0; 9 | ' Initialization ConsensusJunk 10 | ' :self.bwauth_pid_control = True 11 | ' self.group_by_class = False 12 | ' self.use_pid_tgt = False 13 | ' self.use_circ_fails = False 14 | ' self.use_best_ratio = True 15 | ' self.use_desc_bw = True 16 | ' self.use_mercy = False 17 | ' self.guard_sample_rate = GUARD_SAMPLE_RATE 18 | ' self.pid_max = 500.0 19 | ' self.K_p = K_p = 1.0 20 | ' self.T_i = T_i = 0 21 | ' self.T_d = T_d = 0 22 | ' self.T_i_decay = T_i_decay = 0 23 | ' self.K_i = 0 = self.K_i_decay = Kd; 24 | 25 | partition "Initialize relays from consensus (prev_consensus)" { 26 | :ns_list = c.get_network_status(); 27 | 'some ordering i don't understand yet 28 | :ns_list.sort(lambda x, y: int(y.bandwidth/10000.0 - x.bandwidth/10000.0)); 29 | 30 | :prev_consensus = {}; 31 | while (for i in range(0, len(ns_list))?) 32 | :n = ns_list[i]; 33 | :n.list_rank = i; 34 | :n.measured = False; 35 | :prev_consensus["$"+n.idhex] = n; 36 | endwhile 37 | ' If any relay doesn't have bandwidth, exit 38 | } 39 | 40 | partition "Aggregate raw measurements (nodes)" 41 | ' Read measurements 42 | :nodes = {}; 43 | while (for line in bw_file?) 44 | if (line.idhex not in nodes?) then (yes) 45 | :n = Node(); 46 | :nodes[line.idhex] = n; 47 | :n.add_line(line); 48 | endif 49 | endwhile 50 | ' If not measurements, exit 51 | } 52 | 53 | partition "Assign consensus flags" 54 | ' Assign flags (G, M, E) from consensus to measurements 55 | while (for idhex in nodes.iterkeys()?) 56 | if (idhex in prev_consensus?) then (yes) 57 | :nodes[idhex].flags = prev_consensus[idhex].flags; 58 | endif 59 | endwhile 60 | } 61 | 62 | :scaling; 63 | 64 | @enduml 65 | -------------------------------------------------------------------------------- /docs/source/activity_torflow_scaling_simplified.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | title "Torflow measurements scaling with PID control (Per relay scaled bandwidth)." 3 | 4 | ' Own previous bwfile 5 | :prev_votes = VoteSet(); 6 | note right 7 | initialize measurements from previous Bandwidth File 8 | end note 9 | :tot_net_bw = 0; 10 | :; 11 | note right 12 | for every measurement 13 | end note 14 | while (for n in nodes.itervalues()?) 15 | partition "Intialize ratios and pid_error" { 16 | ' Anything not set is initialized to 0 or None 17 | :n.fbw_ratio = n.filt_bw/true_filt_avg[n.node_class()]; 18 | :n.sbw_ratio = n.strm_bw/true_strm_avg[n.node_class()]; 19 | :n.use_bw = n.desc_bw; 20 | :n.pid_error = max(bwstrm_i / bwstrm, bwfilt_i / bwfilt) - 1; 21 | note right 22 | if n.sbw_ratio > n.fbw_ratio: 23 | #assert cs_junk.use_best_ratio == True 24 | n.pid_error = (n.strm_bw - true_strm_avg[n.node_class()]) 25 | / true_strm_avg[n.node_class()] 26 | else: 27 | n.pid_error = (n.filt_bw - true_filt_avg[n.node_class()]) 28 | / true_filt_avg[n.node_class()] 29 | 0 <= n.pid_error <= 500.0 30 | end note 31 | } 32 | if (n.idhex in prev_votes.vote_map?) then (yes) 33 | :; 34 | note right 35 | if n.measured_at > 36 | prev_votes.vote_map[n.idhex].measured_at; 37 | end note 38 | if (measurement newer?) then (yes) 39 | :; 40 | note right 41 | if n.idhex in prev_consensus 42 | and ("Guard" in prev_consensus[n.idhex].flags= 43 | end note 44 | if (in prev_consensus, \nis guard?) then (yes) 45 | :; 46 | note right 47 | if n.idhex not in prev_votes.vote_map 48 | or n.measured_at - prev_votes.vote_map[n.idhex].measured_at 49 | > cs_junk.guard_sample_rate: 50 | # cs_jung.guard_sample_rate = 2*7*24*60*60 # 2wks 51 | end note 52 | if (not exit diff NOT bigger than 2 weeks) then (yes) 53 | :; 54 | note right 55 | \# Use new measurement but not feedback 56 | n.copy_vote(prev_vote.vote_map[n.idhex])); 57 | n.new_bw = n.get_pid_bw(prev_votes.vote_map[n.idhex], 58 | cs_junk.K_p, 59 | cs_junk.K_i, 60 | cs_junk.K_d, 61 | 0.0, False) 62 | end note 63 | :\# self.new_bw = vote.bw * 1000 64 | self.pid_bw = vote.pid_bw 65 | self.pid_error_sum = vote.pid_error_sum 66 | self.pid_delta = vote.pid_delta 67 | 68 | n.new_bw = self.use_bw + self.use_bw * self_pid_error 69 | 70 | n.measured_at = prev_vote.measured_at 71 | n.pid_error = prev_vote.pid_error; 72 | else (no) 73 | :; 74 | note right 75 | # full feedback 76 | n.new_bw = n.get_pid_bw(prev_votes.vote_map[n.idhex], 77 | cs_junk.K_p, 78 | cs_junk.K_i, 79 | cs_junk.K_d, 80 | cs_junk.K_i_decay) 81 | = n.get_pid_bw(prev_votes.vote_map[n.idhex], 82 | 1.0, 0, 0, 0) 83 | end note 84 | :self.prev_error = prev_vote.pid_error 85 | self.pid_bw = self.use_bw 86 | + self.use_bw * self.pid_error 87 | # + self.desc_bw * self.pid_error 88 | self.pid_error_sum = 0 + self.pid_error 89 | n.new_bw = self.pid_bw; 90 | endif 91 | endif 92 | ' No new measurement (in prev bwfile, but havent check consensus), do not vote this round 93 | else (no) 94 | :; 95 | note right 96 | \# Reset values. Don't vote/sample this measurement round. 97 | \# is in the previous bwfile, but haven't check the consensus 98 | n.revert_to_vote(prev_votes.vote_map[n.idhex]) 99 | \# which calls again self.copy_vote(vote) 100 | end note 101 | :self.new_bw = prev_vote.bw*1000 102 | self.pid_bw = prev_vote.pid_bw 103 | self.pid_error_sum = prev_vote.pid_error_sum 104 | self.pid_delta = prev_vote.pid_delta 105 | 106 | self.pid_error = vote.pid_error 107 | self.measured_at = vote.measured_at; 108 | 109 | endif 110 | ' Not in previous bwfile, usually only with authoritites, possibly not in conensus? 111 | else (no) 112 | ' :n.new_bw = n.use_bw + cs_junk.K_p*n.use_bw*n.pid_error = \n 113 | :n.new_bw = n.use_bw + n.use_bw * n.pid_error 114 | n.pid_error_sum = n.pid_error 115 | n.pid_bw = n.new_bw; 116 | endif 117 | ' :n.change = n.new_bw - n.desc_bw; 118 | 119 | ' For capping later 120 | if (n.idhex in prev_consensus) then (yes) 121 | if (prev_consensus[n.idhex].bandwidth != None) then (yes) 122 | :prev_consensus[n.idhex].measured = True; 123 | :tot_net_bw += n.new_bw; 124 | endif 125 | endif 126 | endwhile 127 | while (for n in nodes.itervalues()?) 128 | :cap...; 129 | endwhile 130 | stop 131 | 132 | @enduml 133 | -------------------------------------------------------------------------------- /docs/source/activity_torflow_scaling_simplified1.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | title "Torflow measurements scaling with PID control (Per relay scaled bandwidth)." 3 | 4 | ' Own previous bwfile 5 | :prev_votes = VoteSet(); 6 | :tot_net_bw = 0; 7 | :; 8 | note right 9 | for every measurement 10 | end note 11 | while (for n in nodes.itervalues()?) 12 | partition "Intialize ratios and pid_error" { 13 | ' Anything not set is initialized to 0 or None 14 | :n.fbw_ratio = n.filt_bw/true_filt_avg[n.node_class()]; 15 | :n.sbw_ratio = n.strm_bw/true_strm_avg[n.node_class()]; 16 | :n.use_bw = n.desc_bw; 17 | :n.pid_error = max(n.fbw_ratio, n.sbw_ratio) - 1; 18 | } 19 | if (n.idhex in prev_votes.vote_map \nand not newer measurement?) then (yes) 20 | :self.new_bw = prev_vote.bw*1000 21 | self.measured_at = vote.measured_at; 22 | ' Not in previous bwfile, usually only with authoritites, possibly not in conensus? 23 | else (no) 24 | :n.new_bw = n.use_bw + n.use_bw * n.pid_error; 25 | endif 26 | 27 | ' For capping later 28 | if (n.idhex in prev_consensus \nand prev_consensus[n.idhex].bandwidth != None) then (yes) 29 | :prev_consensus[n.idhex].measured = True 30 | tot_net_bw += n.new_bw; 31 | endif 32 | endwhile 33 | while (for n in nodes.itervalues()?) 34 | :cap...; 35 | endwhile 36 | stop 37 | 38 | @enduml 39 | -------------------------------------------------------------------------------- /docs/source/bandwidth_authorities.rst: -------------------------------------------------------------------------------- 1 | Bandwidth authorities in metrics 2 | ================================= 3 | 4 | Current bandwidth authorities 5 | ----------------------------- 6 | 7 | .. image:: images/bwauth.* 8 | :alt: bandwidth authorities in metrics 9 | 10 | https://metrics.torproject.org/rs.html 11 | 12 | (flag:Authority) 13 | 14 | Bandwidth Authorities - Measured Relays past 7 days 15 | --------------------------------------------------- 16 | 17 | .. image:: images/bwauth_measured_7days.png 18 | :alt: bandwidth measured in the past 7 days 19 | 20 | https://consensus-health.torproject.org/graphs.html 21 | 22 | Bandwidth Authorities - Measured Relays past 90 days 23 | ---------------------------------------------------- 24 | 25 | .. image:: images/bwauth_measured_90days.png 26 | :alt: bandwidth measured in the past 90 days 27 | 28 | https://consensus-health.torproject.org/graphs.html 29 | -------------------------------------------------------------------------------- /docs/source/bandwidth_distribution.rst: -------------------------------------------------------------------------------- 1 | Relays' bandwidth distribution 2 | =================================== 3 | 4 | sbws raw measurements compared to Torflow measurements 5 | ------------------------------------------------------ 6 | 7 | .. image:: images/43710932-ac1eeea8-9960-11e8-9e7e-21fddff2f7a3.png 8 | :alt: sbws and torflow raw measurements distribution 9 | 10 | 11 | .. image:: images/43710933-ac95e0bc-9960-11e8-9aaf-0bb1f83b65e2.png 12 | :alt: sbws and torflow raw measurements distribution 2 13 | 14 | 15 | sbws linear scaling 16 | -------------------- 17 | 18 | Multiply each relay bandwidth by ``7500/median`` 19 | 20 | See bandwidth_file_spec_ appendix B to know how about linear scaling. 21 | 22 | Code: :func:`sbws.lib.v3bwfile.sbws_scale` 23 | 24 | .. image:: images/20180901_163442.png 25 | :alt: sbws linear scaling 26 | 27 | 28 | sbws Torflow scaling 29 | ----------------------- 30 | 31 | See bandwidth_file_spec_ appendix B to know how about torflow scaling. 32 | 33 | Code: :func:`sbws.lib.v3bwfile.torflow_scale` 34 | 35 | .. image:: images/20180901_164014.png 36 | :alt: sbws torflow scaling 37 | 38 | .. _bandwidth_file_spec: https://gitweb.torproject.org/torspec.git/tree/bandwidth-file-spec.txt 39 | -------------------------------------------------------------------------------- /docs/source/code_design.rst: -------------------------------------------------------------------------------- 1 | Code design 2 | ================= 3 | 4 | .. todo:: 5 | - Link to refactor proposal. 6 | - Change this page when refactoring is implemented. 7 | 8 | UML classes diagram 9 | -------------------- 10 | 11 | .. image:: images/classes_original.* 12 | :alt: UML classes diagram 13 | 14 | `classes_original.svg <./_images/classes_original.svg>`_ 15 | 16 | Packages diagram 17 | ----------------- 18 | 19 | .. image:: ./images/packages_sbws.* 20 | :alt: packages diagram 21 | 22 | `packages_sbws.svg <./_images/packages_sbws.svg>`_ 23 | 24 | scanner threads 25 | ---------------- 26 | 27 | - `TorEventListener`: the thread that runs Tor and listens for events. 28 | - ResultDump: the thread that get the measurement results from a queue 29 | every second. 30 | - `multiprocessing.ThreadPool` starts 3 independent threads: 31 | - workers_thread 32 | - tasks_thread 33 | - results_thread 34 | - measurement threads: they execute :func:`sbws.core.scanner.measure_relay` 35 | There'll be a maximum of 3 by default. 36 | 37 | .. image:: images/threads.* 38 | :alt: scanner threads 39 | 40 | Critical sections 41 | ----------------- 42 | 43 | Data types that are read or wrote from the threads. 44 | 45 | .. image:: images/critical_sections.* 46 | :alt: scanner critical sections 47 | :height: 400px 48 | :align: center 49 | 50 | Call graph 51 | -------------- 52 | 53 | Initialization calls to the moment where the measurement threads start. 54 | 55 | .. image:: images/pycallgraph.png 56 | :alt: call graph 57 | :height: 400px 58 | :align: center 59 | 60 | `callgraph.png <./_images/pycallgraph.png>`_ 61 | -------------------------------------------------------------------------------- /docs/source/config.default.ini: -------------------------------------------------------------------------------- 1 | ../../sbws/config.default.ini -------------------------------------------------------------------------------- /docs/source/config.example.ini: -------------------------------------------------------------------------------- 1 | # Minimum configuration that needs to be customized 2 | [scanner] 3 | # A human-readable string with chars in a-zA-Z0-9 to identify your scanner 4 | nickname = sbws_default 5 | # ISO 3166-1 alpha-2 country code. To be edited. 6 | # Default to a non existing country to detect it was not edited. 7 | country = AA 8 | 9 | [destinations] 10 | foo = off 11 | 12 | [destinations.foo] 13 | url = https://example.com/does/not/exist.bin 14 | # ISO 3166-1 alpha-2 country code. To be edited. 15 | # Use ZZ if the destination URL is a domain name and it is in a CDN. 16 | # Default to a non existing country to detect it was not edited. 17 | country = AA 18 | -------------------------------------------------------------------------------- /docs/source/config.log.default.ini: -------------------------------------------------------------------------------- 1 | ../../sbws/config.log.default.ini -------------------------------------------------------------------------------- /docs/source/config.rst: -------------------------------------------------------------------------------- 1 | .. _config_internal: 2 | 3 | Internal code configuration files 4 | ================================== 5 | Sbws has two default config files it reads: one general, and one specific to 6 | logging. 7 | They all get combined internally to the same ``conf`` structure. 8 | 9 | It first reads the config file containing the default values for almost all 10 | options. If you installed sbws in a virtual environment located at /tmp/venv, 11 | then you will probably find the ``config.default.ini`` in a place such as 12 | ``/tmp/venv/lib/python3.5/site-packages/sbws/`` **You should never edit this 13 | file**. The contents of this default config file can be found :ref:`at the 14 | bottom of this page `. 15 | 16 | Second, ``sbws`` will read ``config.log.default.ini``. It will be located in 17 | the same place as the previous file, and **should not be edited** like the 18 | previous file. The contents of this default log config file can be found 19 | :ref:`at the bottom of this page `. Options set here 20 | overwrite options set in the previous config file. 21 | 22 | Sbws then reads your custom config file. By default, it will search for it 23 | in ``~/.sbws.ini``. Options in this file overwrite options set in previously 24 | read config files. 25 | 26 | The user example config file provided by ``sbws`` might look like this. 27 | 28 | .. _init-config: 29 | 30 | .. literalinclude:: /examples/sbws.example.ini 31 | :caption: Example sbws.example.ini 32 | 33 | **No other configuration files are read.** 34 | 35 | .. _default-config: 36 | 37 | Default Configuration 38 | ---------------------- 39 | 40 | .. literalinclude:: config.default.ini 41 | :caption: config.default.ini 42 | 43 | .. _default-log-config: 44 | 45 | If you know how to use 46 | `Python's logging configuration file format`_, 47 | then you can override or add to what is listed here by editing your config file. 48 | 49 | .. literalinclude:: config.log.default.ini 50 | :caption: config.log.default.ini 51 | 52 | .. _Python's logging configuration file format: https://docs.python.org/3.5/library/logging.config.html#logging-config-fileformat 53 | -------------------------------------------------------------------------------- /docs/source/config_tor.rst: -------------------------------------------------------------------------------- 1 | .. _config_tor: 2 | 3 | Internal Tor configuration for the scanner 4 | ------------------------------------------ 5 | 6 | The scanner needs a specific Tor configuration. 7 | The following options are either set when launching Tor or required when 8 | connection to an existing Tor daemon. 9 | 10 | Default configuration: 11 | 12 | - ``SocksPort auto``: To proxy requests over Tor. 13 | - ``CookieAuthentication 1``: The easiest way to authenticate to Tor. 14 | - ``UseEntryGuards 0``: To avoid path bias warnings. 15 | - ``UseMicrodescriptors 0``: Because full server descriptors are needed. 16 | - ``SafeLogging 0``: Useful for logging, since there's no need for anonymity. 17 | - ``LogTimeGranularity 1`` 18 | - ``ProtocolWarnings 1`` 19 | - ``FetchDirInfoEarly 1`` 20 | - ``FetchDirInfoExtraEarly 1``: Respond to `MaxAdvertisedBandwidth` as soon as possible. 21 | - ``FetchUselessDescriptors 1``: Keep fetching descriptors, even when idle. 22 | - ``LearnCircuitBuildTimeout 0``: To keep circuit build timeouts static. 23 | 24 | Configuration that depends on the user configuration file: 25 | 26 | - ``CircuitBuildTimeout ...``: The timeout trying to build a circuit. 27 | - ``DataDirectory ...``: The Tor data directory path. 28 | - ``PidFile ...``: The Tor PID file path. 29 | - ``ControlSocket ...``: The Tor control socket path. 30 | - ``Log notice ...``: The Tor log level and path. 31 | 32 | Configuration that needs to be set on runtime: 33 | 34 | - ``__DisablePredictedCircuits 1``: To build custom circuits. 35 | - ``__LeaveStreamsUnattached 1``: The scanner is attaching the streams itself. 36 | 37 | Configuration that can be set on runtime and fail: 38 | 39 | - ``ConnectionPadding 0``: Useful for avoiding extra traffic, since scanner anonymity is not a goal. 40 | 41 | Currently most of the code that sets this configuration is in :func:`sbws.util.stem.launch_tor` 42 | and the default configuration is ``sbws/globals.py``. 43 | 44 | .. note:: the location of this code is being refactored. 45 | -------------------------------------------------------------------------------- /docs/source/data/critical_sections.dia: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torproject/sbws/fd139b89de20b6d38941f137325e05069995793b/docs/source/data/critical_sections.dia -------------------------------------------------------------------------------- /docs/source/data/scanner.dia: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torproject/sbws/fd139b89de20b6d38941f137325e05069995793b/docs/source/data/scanner.dia -------------------------------------------------------------------------------- /docs/source/data/use_cases_data_sources.dia: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torproject/sbws/fd139b89de20b6d38941f137325e05069995793b/docs/source/data/use_cases_data_sources.dia -------------------------------------------------------------------------------- /docs/source/differences.rst: -------------------------------------------------------------------------------- 1 | .. _differences: 2 | 3 | Differences between Torflow and sbws 4 | ==================================== 5 | 6 | (Last updated 2020-02-18) 7 | 8 | Aggregating measurements and scaling 9 | ------------------------------------ 10 | 11 | Filtering 12 | ~~~~~~~~~ 13 | 14 | Torflow does not exclude relays because of having "few" measurements or "close" 15 | to each other for that relay, like sbws does :ref:`filtering-measurements`. 16 | 17 | However this is currently disabled in sbws. 18 | 19 | Network stream and filtered bandwidth 20 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 21 | 22 | Torflow calculates the network stream and filtered averages by type of relay 23 | :ref:`stream-and-filtered-bandwidth-for-all-relays`, while sbws is not taking 24 | into account the type of relay :ref:`scaling-the-bandwidth-measurements`. 25 | 26 | Values from the previous Bandwidth File 27 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 28 | 29 | sbws is not reading the previous Bandwidth File, but scaling all the values 30 | with the raw measurements. 31 | 32 | Instead, Torflow uses the previous Bandwidth File values in some cases: 33 | 34 | - When a relay measurement is older than the one in the previous 35 | Bandwidth File, it uses all the values from the previous Bandwidth File. 36 | (how is possible that the Bandwidth File would have a newer measurements?):: 37 | 38 | self.new_bw = prev_vote.bw * 1000 39 | 40 | Bandwidth File KeyValues 41 | ~~~~~~~~~~~~~~~~~~~~~~~~ 42 | 43 | sbws does not calculate nor write to the Bandwidth file the ``pid`` variables 44 | and KeyValues that are used in Torflow. Example of Torflow KeyValues not in sbws:: 45 | 46 | measured_at=1613547098 updated_at=1613547098 pid_error=11.275680184 pid_error_sum=11.275680184 pid_bw=23255048 pid_delta=11.0140582849 circ_fail=0.0 47 | 48 | sbws does not have ``measured_at`` and ``updated_at`` either. 49 | 50 | Currently the scaled bandwidth in Torflow does not depend on those extra values 51 | and they seem to be just informative. 52 | -------------------------------------------------------------------------------- /docs/source/documenting.rst: -------------------------------------------------------------------------------- 1 | .. _documenting: 2 | 3 | Installing and building the documentation 4 | ----------------------------------------- 5 | 6 | To build the documentation, extra Python dependencies are needed: 7 | 8 | - Sphinx_ 9 | - recommonmark_ 10 | - Pylint_ (only to update the diagrams) 11 | 12 | To install them from ``sbws``:: 13 | 14 | pip install .[doc] 15 | 16 | To build the documentation as HTML:: 17 | 18 | cd docs/ && make html 19 | 20 | The generated HTML will be in ``docs/build/``. 21 | 22 | To build the manual (``man``) pages:: 23 | 24 | cd docs/ && make man 25 | 26 | The generated man pages will be in ``docs/man/``. 27 | 28 | To build the documentation diagrams:: 29 | 30 | cd docs/ && make umlsvg 31 | 32 | The generated diagrams will be in ``docs/build/_images/``. 33 | 34 | To convert the ``LaTeX`` mathematical formulae to images, extra system dependencies 35 | are needed: 36 | 37 | - Core and Extra Tex_ Live packages 38 | - dvipng_ 39 | 40 | They are included in most distributions. In Debian install them running:: 41 | 42 | apt install texlive-latex-extra dvpipng 43 | 44 | 45 | .. _Sphinx: https://www.sphinx-doc.org 46 | .. _recommonmark: https://recommonmark.readthedocs.io/ 47 | .. _Pylint: https://www.pylint.org/ 48 | .. _Tex: https://www.tug.org/texlive/acquire.html 49 | .. _dvipng: https://www.nongnu.org/dvipng/ -------------------------------------------------------------------------------- /docs/source/examples/sbws.example.ini: -------------------------------------------------------------------------------- 1 | # Minimum configuration that needs to be customized 2 | [scanner] 3 | # A human-readable string with chars in a-zA-Z0-9 to identify your scanner 4 | nickname = sbws_default 5 | # ISO 3166-1 alpha-2 country code where the Web server destination is located. 6 | # Default AA, to detect it was not edited. 7 | country = SN 8 | 9 | [destinations] 10 | # With several destinations, the scanner can continue even if some of them 11 | # fail, which can be caused by a network problem on their side. 12 | # If all of them fail, the scanner will stop, which 13 | # will happen if there is network problem on the scanner side. 14 | 15 | # A destination can be disabled changing `on` by `off` 16 | foo = on 17 | 18 | [destinations.foo] 19 | # the domain and path to the 1GB file. 20 | url = https://example.com/does/not/exist.bin 21 | # Whether to verify or not the TLS certificate. Default True 22 | verify = False 23 | # ISO 3166-1 alpha-2 country code where the Web server destination is located. 24 | # Default AA, to detect it was not edited. 25 | # Use ZZ if the location is unknown (for instance, a CDN). 26 | country = ZZ 27 | 28 | # Number of consecutive times that a destination could not be used to measure 29 | # before stopping to try to use it for a while that by default is 3h. 30 | max_num_failures = 3 31 | 32 | ## The following logging options are set by default. 33 | ## There is no need to change them unless other options are prefered. 34 | ; [logging] 35 | ; # Whether or not to log to a rotating file the directory paths.log_dname 36 | ; to_file = yes 37 | ; # Whether or not to log to stdout 38 | ; to_stdout = yes 39 | ; # Whether or not to log to syslog 40 | ; # NOTE that when sbws is launched by systemd, stdout goes to journal and 41 | ; # syslog. 42 | ; to_syslog = no 43 | 44 | ; # Level to log at. Debug, info, warning, error, critical. 45 | ; # `level` must be set to the lower of all the handler levels. 46 | ; level = debug 47 | ; to_file_level = debug 48 | ; to_stdout_level = info 49 | ; to_syslog_level = info 50 | ; # Format string to use when logging 51 | ; format = %(module)s[%(process)s]: <%(levelname)s> %(message)s 52 | ; # verbose formatter useful for debugging 53 | ; to_file_format = %(asctime)s %(levelname)s %(threadName)s %(filename)s:%(lineno)s - %(funcName)s - %(message)s 54 | ; # Not adding %(asctime)s to to stdout since it'll go to syslog when using 55 | ; # systemd, and it'll have already the date. 56 | ; to_stdout_format = ${format} 57 | ; to_syslog_format = ${format} 58 | 59 | # To disable certificate validation, uncomment the following 60 | # verify = False -------------------------------------------------------------------------------- /docs/source/faq.rst: -------------------------------------------------------------------------------- 1 | Frequently Asked Questions (FAQ) 2 | ================================== 3 | 4 | .. seealso:: :doc:`glossary`. 5 | 6 | How many hops are the circuits used to perform the measurements? 7 | ---------------------------------------------------------------- 8 | 9 | Two hops. 10 | 11 | How are relays selected to be measured? 12 | --------------------------------------- 13 | 14 | The :term:`sbws scanner` periodically refreshes its idea for what relays should 15 | be measured next. It prioritizese the measurement of relays that do not have 16 | recent results. In this way, relays that have just joined the network or have 17 | just come back online after a many-day period of being offline will be measured 18 | before relays that have been online constantly. 19 | 20 | How do sbws scanner results end up in the consensus? 21 | ---------------------------------------------------- 22 | 23 | The :term:`sbws scanner` runs continuously to gather fresh data. 24 | 25 | The :term:`sbws generate` command takes the fresh data and generates a 26 | :term:`v3bw file`. 27 | 28 | The Tor :term:`directory authority` parses the v3bw file and includes bandwidth 29 | information in its vote. 30 | 31 | The authorities take the low-median of the bandwidths for each relay from all 32 | of the :term:`bandwidth authorities ` and use that in the 33 | consensus. 34 | 35 | Does sbws need any open ports? 36 | ------------------------------ 37 | 38 | No. 39 | 40 | How much bandwidth will the sbws scanner use? 41 | --------------------------------------------- 42 | 43 | .. todo:: answer this 44 | 45 | How much bandwidth will the webserver use? 46 | ------------------------------------------ 47 | 48 | .. todo:: answer this 49 | 50 | Should I run my own webserver? Use a CDN? Something else? 51 | --------------------------------------------------------- 52 | 53 | It's up to you. Sbws is very flexible. 54 | 55 | .. todo:: better answer. 56 | -------------------------------------------------------------------------------- /docs/source/generator.rst: -------------------------------------------------------------------------------- 1 | .. _generator: 2 | 3 | How aggregation and scaling works 4 | ================================= 5 | 6 | .. seealso:: :ref:`scanner` (scanner part). 7 | 8 | Every hour, the generator: 9 | 10 | #. Aggregate all the measurements (not older than 6 six days) for every relay. 11 | #. Filter the measurements 12 | #. Scale the measurements 13 | #. Write the bandwidth file 14 | 15 | Source code: :func:`sbws.lib.v3bwfile.V3BWFile.from_results` 16 | 17 | .. _filtering-measurements: 18 | 19 | Filtering the bandwidth measurements 20 | ------------------------------------- 21 | 22 | Each relay bandwidth measurements are selected in the following way: 23 | 24 | #. At least two bandwidth measurements (``Result`` s) MUST have been obtained 25 | within an arbitrary number of seconds (currently one day). 26 | If they are not, the relay MUST NOT be included in the Bandwith File. 27 | #. The measurements than are are older than an arbitrary number of senconds 28 | in the past MUST be discarded. 29 | Currently this number is the same as ``data_period`` (5 days) when not 30 | scaling as Torflow and 28 days when scaling as Torflow. 31 | 32 | If the number of relays to include in the Bandwidth File are less than 33 | a percententage (currently 60%) than the number of relays in the consensus, 34 | additional Header Lines MUST be added (see XXX) to the Bandwith File and the 35 | relays SHOULD NOT be included. 36 | 37 | .. image:: ./images/activity_aggr_file.svg 38 | 39 | .. image:: ./images/activity_aggr_lines.svg 40 | 41 | .. _scaling-the-bandwidth-measurements: 42 | 43 | Scaling the bandwidth measurements 44 | ------------------------------------ 45 | 46 | Consensus bandwidth obtained by new implementations MUST be comparable to the 47 | consensus bandwidth, therefore they MUST implement torflow_scaling_. 48 | 49 | The bandwidth_file_spec_ appendix B describes torflow scaling and a linear 50 | scaling method. 51 | 52 | .. image:: ./images/activity_scaling_as_torflow.svg 53 | 54 | .. seealso:: :ref:`torflow_aggr` and :ref:`differences`. 55 | 56 | 57 | Writing the bandwidth file 58 | --------------------------- 59 | 60 | The bandwidth file format is defined in the bandwidth_file_spec_. 61 | 62 | 63 | .. _torflow_scaling: https://gitweb.torproject.org/torflow.git/tree/NetworkScanners/BwAuthority/README.spec.txt#n298 64 | .. _bandwidth_file_spec: https://gitweb.torproject.org/torspec.git/tree/bandwidth-file-spec.txt 65 | -------------------------------------------------------------------------------- /docs/source/glossary.rst: -------------------------------------------------------------------------------- 1 | Glossary 2 | ========== 3 | 4 | .. glossary:: 5 | 6 | directory authority 7 | a special-purpose relay that maintains a list of currently-running 8 | relays and periodically publishes a consensus together with the other 9 | directory authorities. [#]_ 10 | 11 | bandwidth authority 12 | A :term:`directory authority` that runs a :term:`scanner` and a 13 | :term:`generator` or obtain :term:`bandwidth list file` s from a 14 | :term:`generator`. 15 | 16 | scanner 17 | Term to refer to the process that measures the relays' bandwidth. 18 | It is also called :term:`generator` when it is the same tool that is 19 | used to generate :term:`bandwidth list file` s. 20 | 21 | generator 22 | Term to refer to the tool that generates the 23 | :term:`bandwidth list file` s. Often used as a synomym for 24 | :term:`scanner`. 25 | 26 | bandwidth list file 27 | The file generated by :term:`generator` s that is read by the 28 | :term:`directory authority` s and included in their votes. 29 | See `bandwidth-file-spec.txt `_ 30 | to know about the file specification. 31 | 32 | sbws scanner 33 | The ``sbws`` command used to run ``sbws`` as a :term:`scanner`. 34 | 35 | sbws generate 36 | The ``sbws`` command used to run ``sbws`` as a :term:`generator`. 37 | 38 | v3bw file 39 | The term used by ``sbws`` to refer to :term:`bandwidth list file` 40 | v1.1.0. 41 | 42 | .. literalinclude:: v3bw.txt 43 | :caption: A v3bw file 44 | 45 | destination 46 | The term used by ``sbws`` to refer to a Web server where the 47 | :term:`scanner` request files to perform the bandwith measurements. 48 | 49 | .. [#] https://metrics.torproject.org/glossary.html 50 | -------------------------------------------------------------------------------- /docs/source/how_works.rst: -------------------------------------------------------------------------------- 1 | .. _scanner: 2 | 3 | How sbws works 4 | ============== 5 | 6 | Overview 7 | --------- 8 | 9 | .. The following text is part of the introduction in the README, but rst 10 | formatted. 11 | 12 | The :term:`scanner` measures the bandwidth of each relay in the Tor network 13 | (except the directory authorities) by creating a two hops circuit 14 | with the relay. It then measures the bandwidth by downloading data 15 | from a :term:`destination` Web Server and stores the measurements. 16 | 17 | The :term:`generator` read the measurements, aggregates, filters and 18 | scales them using torflow's scaling method. 19 | 20 | Then it generates a :term:`bandwidth list file` that is read 21 | by a :term:`directory authority` to report relays’ bandwidth in its vote. 22 | 23 | .. image:: ./images/scanner.svg 24 | :height: 200px 25 | :align: center 26 | 27 | .. image:: ./images/dirauths_bwauths.png 28 | 29 | Intialization 30 | -------------- 31 | 32 | .. At some point it should be able to get environment variables 33 | 34 | #. Parse the command line arguments and configuration files. 35 | #. Launch a Tor thread with an specific configuration or connect to a running 36 | Tor daemon that is running with a suitable configuration. 37 | #. Obtain the list of relays in the Tor network from the Tor consensus and 38 | descriptor documents. 39 | #. Read and parse the old bandwidth measurements stored in the file system. 40 | #. Select a subset of the relays to be measured next, ordered by: 41 | 42 | #. relays not measured. 43 | #. measurements age. 44 | 45 | .. image:: ./images/use_cases_data_sources.svg 46 | :alt: data sources 47 | :height: 200px 48 | :align: center 49 | 50 | Classes used in the initialization: 51 | 52 | .. image:: ./images/use_cases_classes.svg 53 | :alt: classes initializing data 54 | :height: 300px 55 | :align: center 56 | 57 | Source code: :func:`sbws.core.scanner.run_speedtest` 58 | 59 | Measuring relays 60 | ----------------- 61 | 62 | #. For every relay: 63 | #. Select a second relay to build a Tor circuit. 64 | #. Build the circuit. 65 | #. Make HTTPS GET requests to the Web server over the circuit. 66 | #. Store the time the request took and the amount of bytes requested. 67 | 68 | .. image:: ./images/activity_all.svg 69 | :alt: activity measuring relays 70 | :height: 300px 71 | :align: center 72 | 73 | Source code: :func:`sbws.core.scanner.measure_relay` 74 | 75 | Measuring a relay 76 | ~~~~~~~~~~~~~~~~~ 77 | 78 | .. image:: ./images/activity_measure.svg 79 | :alt: activity measuring a relay 80 | :height: 300px 81 | :align: center 82 | 83 | Source code: :func:`sbws.core.scanner.measure_relay` 84 | 85 | Selecting a second relay 86 | ------------------------ 87 | 88 | #. If the relay to measure is an exit, use it as an exit and obtain the 89 | non-exits. 90 | #. If the relay to measure is not an exit, use it as first hop and obtain 91 | the exits. 92 | #. From non-exits or exits, select one randomly from the ones that have 93 | double consensus bandwidth than the relay to measure. 94 | #. If there are no relays that satisfy this, lower the required bandwidth. 95 | 96 | .. image:: ./images/activity_second_relay.svg 97 | :alt: activity select second relay 98 | :height: 400px 99 | :align: center 100 | 101 | Source code: :func:`sbws.core.scanner.measure_relay` 102 | 103 | Selecting the data to download 104 | ------------------------------- 105 | 106 | #. While the downloaded data is smaller than 1GB or the number of download 107 | is minor than 5: 108 | #. Randomly, select a 16MiB range. 109 | #. If it takes less than 5 seconds, select a bigger range and don't keep any 110 | information. 111 | #. If it takes more than 10 seconds, select an smaller range and don't keep any 112 | information. 113 | #. Store the number of bytes downloaded and the time it took. 114 | 115 | Source code: :func:`sbws.core.scanner._should_keep_result` 116 | 117 | Writing the measurements to the filesystem 118 | ------------------------------------------- 119 | 120 | For every measured relay, the measurement result is put in a queue. 121 | There's an independent thread getting measurements from the queue every second. 122 | Every new measurement is appended to a file as a json line 123 | (but the file itself is not json!). 124 | The file is named with the current date. Every day a new file is created. 125 | 126 | Source code: :func:`sbws.lib.resultdump.ResultDump.enter` 127 | 128 | .. seealso:: :ref:`generator`. 129 | 130 | .. _torflow: https://gitweb.torproject.org/torflow.git 131 | .. _stem: https://stem.torproject.org 132 | .. https://github.com/requests/requests/issues/4885 133 | .. _requests: http://docs.python-requests.org/ 134 | .. _peerflow: https://www.nrl.navy.mil/itd/chacs/sites/www.nrl.navy.mil.itd.chacs/files/pdfs/16-1231-4353.pdf 135 | -------------------------------------------------------------------------------- /docs/source/images/20180901_163442.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torproject/sbws/fd139b89de20b6d38941f137325e05069995793b/docs/source/images/20180901_163442.png -------------------------------------------------------------------------------- /docs/source/images/20180901_164014.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torproject/sbws/fd139b89de20b6d38941f137325e05069995793b/docs/source/images/20180901_164014.png -------------------------------------------------------------------------------- /docs/source/images/20210111_consensushealth_bwauths.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torproject/sbws/fd139b89de20b6d38941f137325e05069995793b/docs/source/images/20210111_consensushealth_bwauths.png -------------------------------------------------------------------------------- /docs/source/images/43710932-ac1eeea8-9960-11e8-9e7e-21fddff2f7a3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torproject/sbws/fd139b89de20b6d38941f137325e05069995793b/docs/source/images/43710932-ac1eeea8-9960-11e8-9e7e-21fddff2f7a3.png -------------------------------------------------------------------------------- /docs/source/images/43710933-ac95e0bc-9960-11e8-9aaf-0bb1f83b65e2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torproject/sbws/fd139b89de20b6d38941f137325e05069995793b/docs/source/images/43710933-ac95e0bc-9960-11e8-9aaf-0bb1f83b65e2.png -------------------------------------------------------------------------------- /docs/source/images/advertised_bandwidth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torproject/sbws/fd139b89de20b6d38941f137325e05069995793b/docs/source/images/advertised_bandwidth.png -------------------------------------------------------------------------------- /docs/source/images/bwauth_measured_7days.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torproject/sbws/fd139b89de20b6d38941f137325e05069995793b/docs/source/images/bwauth_measured_7days.png -------------------------------------------------------------------------------- /docs/source/images/bwauth_measured_90days.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torproject/sbws/fd139b89de20b6d38941f137325e05069995793b/docs/source/images/bwauth_measured_90days.png -------------------------------------------------------------------------------- /docs/source/images/dirauths_bwauths.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torproject/sbws/fd139b89de20b6d38941f137325e05069995793b/docs/source/images/dirauths_bwauths.png -------------------------------------------------------------------------------- /docs/source/images/pycallgraph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torproject/sbws/fd139b89de20b6d38941f137325e05069995793b/docs/source/images/pycallgraph.png -------------------------------------------------------------------------------- /docs/source/images/torperf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torproject/sbws/fd139b89de20b6d38941f137325e05069995793b/docs/source/images/torperf.png -------------------------------------------------------------------------------- /docs/source/images/totalcw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torproject/sbws/fd139b89de20b6d38941f137325e05069995793b/docs/source/images/totalcw.png -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. simple-bw-scanner documentation master file, created by 2 | sphinx-quickstart on Fri Mar 23 16:20:02 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to Simple Bandwidth Scanner's documentation! 7 | ==================================================== 8 | 9 | User main documentation 10 | ------------------------ 11 | 12 | Included in the 13 | `repository root `_ 14 | and in ``sbws`` Debian package: 15 | 16 | .. toctree:: 17 | :maxdepth: 1 18 | 19 | README 20 | INSTALL 21 | DEPLOY 22 | CHANGELOG 23 | AUTHORS 24 | man_sbws 25 | man_sbws.ini 26 | 27 | .. _dev_doc: 28 | 29 | Developer/technical documentation 30 | ---------------------------------- 31 | 32 | Included in the 33 | `docs directory `_ and in 34 | ``sbws-doc`` Debian package: 35 | 36 | .. toctree:: 37 | :maxdepth: 1 38 | 39 | contributing 40 | testing 41 | documenting 42 | how_works 43 | generator 44 | torflow_aggr 45 | differences 46 | code_design 47 | state 48 | config 49 | config_tor 50 | sbws 51 | implementation 52 | bandwidth_distribution 53 | tor_bandwidth_files 54 | bandwidth_authorities 55 | monitoring_bandwidth 56 | roadmap 57 | glossary 58 | faq 59 | 60 | Proposals: 61 | 62 | .. toctree:: 63 | :maxdepth: 1 64 | :glob: 65 | 66 | proposals/* 67 | 68 | Indices and tables 69 | ------------------- 70 | 71 | * :ref:`genindex` 72 | * :ref:`modindex` 73 | * :ref:`search` 74 | -------------------------------------------------------------------------------- /docs/source/man_sbws.rst: -------------------------------------------------------------------------------- 1 | Simple Bandwidth Scanner - SBWS(1) 2 | =================================== 3 | 4 | SYNOPSIS 5 | -------- 6 | 7 | sbws [**Optional arguments**] [**Positional arguments**] 8 | 9 | sbws [**-h**] [**--version**] 10 | [**--log-level** {**debug,info,warning,error,critical**}] 11 | [**-c** CONFIG] {**cleanup,scanner,generate,init,stats**} 12 | 13 | DESCRIPTION 14 | ----------- 15 | 16 | Tor bandwidth scanner that generates bandwidth measurements files to be read by 17 | the Directory Authorities. 18 | 19 | The **scanner** requires a configuration file (see **sbws.ini** (5)) with a 20 | with a '[destinations]' section. 21 | 22 | **sbws** can be run a python script or a system service. 23 | The later is recommended. 24 | 25 | The default locations of the files that **sbws** reads or generate depend on 26 | on how it is run. 27 | See the section **FILES** to know which are the default locations. 28 | 29 | OPTIONS 30 | ------- 31 | 32 | Positional arguments 33 | ~~~~~~~~~~~~~~~~~~~~ 34 | 35 | {**cleanup,scanner,generate,init,stats**} 36 | 37 | These arguments can have additional optional arguments. 38 | To obtain information about them, run: 'sbws --help'. 39 | 40 | Optional arguments 41 | ~~~~~~~~~~~~~~~~~~ 42 | 43 | -h, --help 44 | Show help message and exit. 45 | 46 | --version 47 | Show **sbws** version and exit. 48 | 49 | --log-level {debug,info,warning,error,critical} 50 | Override the sbws log level (default: info). 51 | 52 | -c CONFIG, --config CONFIG 53 | Path to a custom configuration file. 54 | 55 | EXAMPLES 56 | -------- 57 | 58 | sbws scanner 59 | Run the scanner using **sbws** defaults. 60 | 61 | sbws -c ~/.sbwsrc scanner 62 | Run the scanner using the configuration file in `~/.sbwsrc` 63 | 64 | sbws --log-level debug generate 65 | Generate v3bw file in the default v3bw directory. 66 | 67 | sbws cleanup 68 | Cleanup datadir and v3bw files older than XX in the default v3bw directory. 69 | 70 | FILES 71 | ----- 72 | 73 | In the following list, the first path is the default location when running 74 | **sbws** as an script, the second path is the default location when running 75 | **sbws** as a system service. 76 | 77 | **$HOME/.sbws.ini** or **/etc/sbws/sbws.ini** 78 | Location where **sbws** searchs for a custom configuration file, when the 79 | option **--config** is not provided. 80 | 81 | **$HOME/.sbws** or **/var/lib/sbws** 82 | Location where **sbws** writes/reads measurement data files, 83 | bandwidth list files and **tor** process data. 84 | 85 | Under this directory, **sbws** creates the following subdirectories: 86 | 87 | datadir 88 | Raw results generated by the ``sbws scanner``. 89 | Other commands (such as ``generate`` and ``stats``) read results from 90 | this directory. 91 | 92 | log 93 | Log files generated by ``sbws``, when logging to a file is configured 94 | (see **sbws.ini**). 95 | 96 | v3bw 97 | Bandwidth files generated by ``sbws generate``. These are the files 98 | read by the Tor directory authorities. 99 | 100 | tor 101 | Data generated by the **tor** process launched by **sbws**. 102 | 103 | **$HOME/.sbws/tor** or **/run/sbws/tor** 104 | Location where the **tor** process launched by ``sbws scanner`` stores 105 | temporal files, like Unix domain sockets. 106 | 107 | SEE ALSO 108 | --------- 109 | 110 | **sbws.ini** (5), https://sbws.readthedocs.org, 111 | https://gitweb.torproject.org/torspec.git/tree/bandwidth-file-spec.txt, 112 | **tor** (1). 113 | 114 | BUGS 115 | ---- 116 | 117 | Please report bugs at https://gitlab.torproject.org/tpo/network-health/sbws/-/issues/. 118 | -------------------------------------------------------------------------------- /docs/source/monitoring_bandwidth.rst: -------------------------------------------------------------------------------- 1 | Monitoring bandwidth changes in the Tor Network 2 | ================================================ 3 | 4 | Bandwidth authorities timeline 5 | ------------------------------ 6 | 7 | Events that can affect the data generated by the bwauths: 8 | 9 | https://gitlab.torproject.org/tpo/network-health/sbws/-/wikis/bandwidth%20authorities%20timeline 10 | 11 | This page might be moved to a different location. 12 | 13 | Bwauths number of measured relays 14 | --------------------------------- 15 | 16 | It should be approximately equal for all bwauths. 17 | 18 | .. image:: images/20210111_consensushealth_bwauths.png 19 | :alt: bwauths measured relays 20 | 21 | https://consensus-health.torproject.org/graphs.html#votedaboutgraphs 22 | 23 | http://tgnv2pssfumdedyw.onion/graphs.html#votedaboutgraphs 24 | 25 | Total consensus weights across bandwidth authorities 26 | ---------------------------------------------------- 27 | 28 | It should be approximately equal for all bwauths. 29 | 30 | .. image:: images/totalcw.png 31 | :alt: total consensus weight 32 | 33 | 34 | ​https://metrics.torproject.org/totalcw.html 35 | 36 | 37 | Not measured relays and descriptors and consensus updates 38 | --------------------------------------------------------- 39 | 40 | Run the tool https://gitlab.torproject.org/juga/bwauthealth. 41 | 42 | 43 | Total bandwidth 44 | --------------- 45 | 46 | Should not decrease. 47 | 48 | .. image:: images/advertised_bandwidth.png 49 | :alt: advertised bandwidth 50 | 51 | 52 | ​https://metrics.torproject.org/bandwidth-flags.html 53 | 54 | 55 | Time to download a file 56 | ----------------------- 57 | 58 | Should not increase. 59 | 60 | .. image:: images/torperf.png 61 | :alt: torperf 62 | 63 | 64 | ​https://metrics.torproject.org/torperf.html 65 | -------------------------------------------------------------------------------- /docs/source/roadmap.rst: -------------------------------------------------------------------------------- 1 | .. _roadmap: 2 | 3 | Roadmap 4 | ======== 5 | 6 | Release 1.0.0 7 | -------------- 8 | 9 | Autum 2018 10 | - Minimal Viable Product (MVP) 11 | 12 | Release 1.1.0 13 | -------------- 14 | 15 | TBD -------------------------------------------------------------------------------- /docs/source/sbws.core.rst: -------------------------------------------------------------------------------- 1 | sbws.core package 2 | ===================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | sbws.core.cleanup module 8 | ~~~~~~~~~~~~~~~~~~~~~~~~ 9 | 10 | .. automodule:: sbws.core.cleanup 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | sbws.core.generate module 16 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 17 | 18 | .. automodule:: sbws.core.generate 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | sbws.core.scanner module 24 | ~~~~~~~~~~~~~~~~~~~~~~~~ 25 | 26 | .. automodule:: sbws.core.scanner 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | sbws.core.stats module 32 | ~~~~~~~~~~~~~~~~~~~~~~ 33 | 34 | .. automodule:: sbws.core.stats 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: sbws.core 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /docs/source/sbws.lib.rst: -------------------------------------------------------------------------------- 1 | sbws.lib package 2 | ================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | sbws.lib.circuitbuilder module 8 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 9 | 10 | .. automodule:: sbws.lib.circuitbuilder 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | sbws.lib.relaylist module 16 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 17 | 18 | .. automodule:: sbws.lib.relaylist 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | sbws.lib.relayprioritizer module 24 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 25 | 26 | .. automodule:: sbws.lib.relayprioritizer 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | sbws.lib.resultdump module 32 | ~~~~~~~~~~~~~~~~~~~~~~~~~~ 33 | 34 | .. automodule:: sbws.lib.resultdump 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | sbws.lib.v3bwfile module 40 | ~~~~~~~~~~~~~~~~~~~~~~~~~~ 41 | 42 | .. automodule:: sbws.lib.v3bwfile 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | Module contents 48 | --------------- 49 | 50 | .. automodule:: sbws.lib 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | -------------------------------------------------------------------------------- /docs/source/sbws.rst: -------------------------------------------------------------------------------- 1 | Package API 2 | ================= 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | sbws.core 10 | sbws.lib 11 | sbws.util 12 | 13 | Submodules 14 | ---------- 15 | 16 | sbws.globals module 17 | ------------------- 18 | 19 | .. automodule:: sbws.globals 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /docs/source/sbws.util.rst: -------------------------------------------------------------------------------- 1 | sbws.util package 2 | ================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | sbws.util.config module 8 | ~~~~~~~~~~~~~~~~~~~~~~~ 9 | 10 | .. automodule:: sbws.util.config 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | sbws.util.filelock module 16 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 17 | 18 | .. automodule:: sbws.util.filelock 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | sbws.util.parser module 24 | ~~~~~~~~~~~~~~~~~~~~~~~ 25 | 26 | .. automodule:: sbws.util.parser 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | sbws.util.state module 32 | ~~~~~~~~~~~~~~~~~~~~~~ 33 | 34 | .. automodule:: sbws.util.state 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | sbws.util.stem module 40 | ~~~~~~~~~~~~~~~~~~~~~ 41 | 42 | .. automodule:: sbws.util.stem 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | sbws.util.userquery module 48 | ~~~~~~~~~~~~~~~~~~~~~~~~~~ 49 | 50 | .. automodule:: sbws.util.userquery 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | 56 | Module contents 57 | --------------- 58 | 59 | .. automodule:: sbws.util 60 | :members: 61 | :undoc-members: 62 | :show-inheritance: 63 | -------------------------------------------------------------------------------- /docs/source/state.rst: -------------------------------------------------------------------------------- 1 | The ``state.dat`` file 2 | ====================== 3 | 4 | This file contains state that multiple sbws commands may want access to and 5 | that needs to persist across processes. Both read and write access to this file 6 | is wrapped in the ``State`` class, allowing for safe concurrent access: the 7 | file is locked before reading or writing, and (for now) only simple data types 8 | are allowed so we can be sure to update the state file on disk every time the 9 | state is modified in memory. 10 | 11 | At the time of writing, the following fields can exist in the state file. 12 | 13 | ``scanner_started`` 14 | ------------------- 15 | 16 | The last time ``sbws scanner`` was started. 17 | 18 | - **Producer**: ``sbws scanner``, once at startup. 19 | 20 | - **Consumer**: ``sbws generate``, once each time it is ran. 21 | 22 | Code: :class:`sbws.util.state.State` -------------------------------------------------------------------------------- /docs/source/testing.rst: -------------------------------------------------------------------------------- 1 | .. _testing: 2 | 3 | Installing tests dependencies and running tests 4 | ================================================== 5 | 6 | To run the tests, extra Python depenencies are needed: 7 | 8 | - Flake8_ 9 | - tox_ 10 | - pytest_ 11 | - coverage_ 12 | 13 | To install them from ``sbws`` :: 14 | 15 | pip install .[dev] && pip install .[test] 16 | 17 | To run the tests:: 18 | 19 | tox 20 | 21 | .. _Flake8: https://flake8.readthedocs.io/ 22 | .. _pytest: https://docs.pytest.org/ 23 | .. _tox: https://tox.readthedocs.io 24 | .. _Coverage: https://coverage.readthedocs.io/ -------------------------------------------------------------------------------- /docs/source/threads.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | start 4 | 5 | :MainThread; 6 | fork 7 | :TorEventListener; 8 | fork again 9 | :ResultDump; 10 | fork again 11 | :workers_thread; 12 | fork again 13 | :tasks_thread; 14 | fork again 15 | :resutls_tread; 16 | fork again 17 | while (threads < 3?) 18 | :measure_relay; 19 | endwhile 20 | end fork 21 | 22 | 23 | @enduml -------------------------------------------------------------------------------- /docs/source/tor_bandwidth_files.rst: -------------------------------------------------------------------------------- 1 | How bandwidth files are shown in the Tor network 2 | ================================================= 3 | 4 | Directory authorities' votes 5 | ----------------------------- 6 | 7 | moria, using Tor 0.3.5.7: 8 | 9 | .. code:: text 10 | 11 | bandwidth-file-headers timestamp=1548181637 12 | 13 | https://collector.torproject.org/recent/relay-descriptors/votes/ 14 | 15 | To appear in Tor v0.4.1.x: 16 | 17 | .. code:: text 18 | 19 | bandwidth-file-digest sha256=01234567890123456789abcdefghijkl 20 | 21 | https://gitlab.torproject.org/tpo/core/tor/-/issues/26698 22 | 23 | Directory authorities' bandwidth file URL 24 | ----------------------------------------- 25 | 26 | To appear in Tor v0.4.1.x: 27 | 28 | .. code:: text 29 | 30 | /tor/status-vote/next/bandwidth.z 31 | 32 | https://gitlab.torproject.org/tpo/core/tor/-/issues/21377 33 | -------------------------------------------------------------------------------- /docs/source/v3bw.txt: -------------------------------------------------------------------------------- 1 | 1524159868 2 | version=0.1.0 3 | node_id=$1BA71540E05D18401B65B553C35DA71992B9E488 bw=6941170 nick=exit2 rtt=20 time=1524107856 4 | node_id=$189442066BEF15F777738E4E063B7BE0285EA0D9 bw=6855121 nick=exit3 rtt=19 time=1524107855 5 | node_id=$076697F1272A92110AB82226699E62C4EFD49766 bw=6810514 nick=relay4 rtt=20 time=1524107855 6 | node_id=$57BD9518CCC40874D969F0784922EF8B89EB9707 bw=6693692 nick=relay7 rtt=20 time=1524107837 7 | node_id=$B5B33BCBC8C779BFE7B319E0CC3EA6E52EA355EA bw=6653275 nick=relay3 rtt=38 time=1524107847 8 | node_id=$514326DD0EA15A41F1E7840C421A06CCCB2E39FA bw=6614808 nick=exit1 rtt=20 time=1524107837 9 | node_id=$4E5FBF937A4C1D4F9211780BF700E70E30004910 bw=6593946 nick=relay1 rtt=19 time=1524107855 10 | node_id=$D17B78F14F66F9F29686B37A78B77F6AC17DCE92 bw=6483024 nick=relay6 rtt=24 time=1524107848 11 | node_id=$883505B618A0F14EE6136F1451CD4F00760C105F bw=6257421 nick=relay5 rtt=20 time=1524107865 12 | node_id=$654E99AF0EAFA05DCD576C8607F15F3B076C53C8 bw=6069373 nick=relay2 rtt=19 time=1524107860 13 | -------------------------------------------------------------------------------- /sbws/__init__.py: -------------------------------------------------------------------------------- 1 | from ._version import get_versions 2 | __version__ = get_versions()['version'] 3 | del get_versions 4 | 5 | import threading # noqa 6 | 7 | from . import globals # noqa 8 | 9 | 10 | class Settings: 11 | """Singleton settings for all the packages. 12 | This way change settings can be seen by all the packages that import it. 13 | 14 | It lives in ``__init__.py`` to leave open the possibility of having a 15 | ``settings.py`` module for user settings. 16 | 17 | .. note:: After refactoring, globals should only have constants. 18 | Any other variable that needs to be modified when initializing 19 | should be initialized here. 20 | 21 | """ 22 | def __init__(self): 23 | # update this dict from globals (but only for ALL_CAPS settings) 24 | for setting in dir(globals): 25 | if setting.isupper(): 26 | setattr(self, setting, getattr(globals, setting)) 27 | self.end_event = threading.Event() 28 | 29 | def init_http_headers(self, nickname, uuid, tor_version): 30 | self.HTTP_HEADERS['Tor-Bandwidth-Scanner-Nickname'] = nickname 31 | self.HTTP_HEADERS['Tor-Bandwidth-Scanner-UUID'] = uuid 32 | self.HTTP_HEADERS['User-Agent'] += tor_version 33 | 34 | def set_end_event(self): 35 | self.end_event.set() 36 | 37 | 38 | settings = Settings() # noqa 39 | -------------------------------------------------------------------------------- /sbws/config.log.default.ini: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys = root,sbws 3 | 4 | [handlers] 5 | keys = to_file,to_stdout,to_syslog 6 | 7 | [formatters] 8 | keys = to_file,to_stdout,to_syslog 9 | 10 | [logger_root] 11 | level = WARNING 12 | handlers = to_file 13 | propagate = 1 14 | qualname=root 15 | 16 | [logger_sbws] 17 | propagate = 0 18 | qualname=sbws 19 | 20 | [handler_to_stdout] 21 | class = StreamHandler 22 | formatter = to_stdout 23 | args = (sys.stdout,) 24 | 25 | [handler_to_file] 26 | class = handlers.RotatingFileHandler 27 | formatter = to_file 28 | args = ('/dev/null', ) 29 | 30 | # for logging to system log 31 | [handler_to_syslog] 32 | class=handlers.SysLogHandler 33 | formatter=to_syslog 34 | args = ('/dev/log',) 35 | 36 | [formatter_to_stdout] 37 | # format date as syslog and journal 38 | datefmt = %b %d %H:%M:%S 39 | 40 | [formatter_to_file] 41 | datefmt = %b %d %H:%M:%S 42 | 43 | [formatter_to_syslog] 44 | -------------------------------------------------------------------------------- /sbws/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torproject/sbws/fd139b89de20b6d38941f137325e05069995793b/sbws/core/__init__.py -------------------------------------------------------------------------------- /sbws/core/bwfile_health.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """""" 3 | import argparse 4 | 5 | from sbws.lib.bwfile_health import BwFile 6 | 7 | 8 | def main(): 9 | parser = argparse.ArgumentParser() 10 | parser.add_argument("-f", "--file-path", help="Bandwidth file path.") 11 | 12 | args = parser.parse_args() 13 | 14 | header_health = BwFile.load(args.file_path) 15 | header_health.report 16 | 17 | 18 | if __name__ == "__main__": 19 | main() 20 | -------------------------------------------------------------------------------- /sbws/lib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torproject/sbws/fd139b89de20b6d38941f137325e05069995793b/sbws/lib/__init__.py -------------------------------------------------------------------------------- /sbws/lib/circuitbuilder.py: -------------------------------------------------------------------------------- 1 | from stem import CircuitExtensionFailed, InvalidRequest, ProtocolError, Timeout 2 | from stem import InvalidArguments, ControllerError, SocketClosed 3 | import logging 4 | 5 | log = logging.getLogger(__name__) 6 | 7 | 8 | def valid_circuit_length(path): 9 | return 0 < len(path) <= 8 10 | 11 | 12 | class CircuitBuilder: 13 | ''' The CircuitBuilder interface. 14 | 15 | Subclasses must implement their own build_circuit() function. 16 | Subclasses may keep additional state if they'd find it helpful. 17 | 18 | The primary way to use a CircuitBuilder of any type is to simply create it 19 | and then call cb.build_circuit(...) with any options that your 20 | CircuitBuilder type needs. 21 | 22 | It might be good practice to close circuits as you find you no longer need 23 | them, but CircuitBuilder will keep track of existing circuits and close 24 | them when it is deleted. 25 | ''' 26 | # XXX: In new major version, remove args and conf, they are not used. 27 | def __init__(self, args, conf, controller, relay_list=None, 28 | close_circuits_on_exit=True): 29 | self.controller = controller 30 | self.built_circuits = set() 31 | self.close_circuits_on_exit = close_circuits_on_exit 32 | self.circuit_timeout = conf.getint('general', 'circuit_timeout') 33 | 34 | def close_circuit(self, circ_id): 35 | try: 36 | self.controller.close_circuit(circ_id) 37 | # SocketClosed will be raised when stopping sbws 38 | except (InvalidArguments, InvalidRequest, SocketClosed) as e: 39 | log.debug(e) 40 | self.built_circuits.discard(circ_id) 41 | 42 | def _build_circuit_impl(self, path): 43 | """ 44 | :returns tuple: circuit id if the circuit was built, error if there 45 | was an error building the circuit. 46 | """ 47 | if not valid_circuit_length(path): 48 | return None, "Can not build a circuit, invalid path." 49 | c = self.controller 50 | timeout = self.circuit_timeout 51 | fp_path = '[' + ' -> '.join([p for p in path]) + ']' 52 | log.debug('Building %s', fp_path) 53 | try: 54 | circ_id = c.new_circuit( 55 | path, await_build=True, timeout=timeout) 56 | except (InvalidRequest, CircuitExtensionFailed, 57 | ProtocolError, Timeout, SocketClosed) as e: 58 | return None, str(e) 59 | return circ_id, None 60 | 61 | def __del__(self): 62 | c = self.controller 63 | if not self.close_circuits_on_exit: 64 | return 65 | for circ_id in self.built_circuits: 66 | try: 67 | c.get_circuit(circ_id, default=None) 68 | try: 69 | c.close_circuit(circ_id) 70 | except (InvalidArguments, InvalidRequest): 71 | pass 72 | except (ControllerError, InvalidArguments) as e: 73 | log.exception("Exception trying to get circuit to delete: %s", 74 | e) 75 | self.built_circuits.clear() 76 | 77 | 78 | # In a future refactor, remove this class, since sbws chooses the relays to 79 | # build the circuit, the relays are not just choosen as random as this class 80 | # does. 81 | class GapsCircuitBuilder(CircuitBuilder): 82 | """Same as ``CircuitBuilder`` but implements build_circuit.""" 83 | def __init__(self, *a, **kw): 84 | super().__init__(*a, **kw) 85 | 86 | def build_circuit(self, path): 87 | """Return parent class build circuit method. 88 | 89 | Since sbws is only building 2 hop paths, there is no need to add random 90 | relays to the path, or convert back and forth between fingerprint and 91 | ``Relay`` objects. 92 | 93 | """ 94 | return self._build_circuit_impl(path) 95 | -------------------------------------------------------------------------------- /sbws/lib/heartbeat.py: -------------------------------------------------------------------------------- 1 | """ 2 | Classes and functions to implement a heartbeat system to monitor the progress. 3 | """ 4 | import logging 5 | import time 6 | 7 | from ..util.state import State 8 | 9 | 10 | log = logging.getLogger(__name__) 11 | 12 | 13 | class Heartbeat(object): 14 | """ 15 | Tracks current status of sbws and is capable of printing periodic 16 | information about the current state 17 | """ 18 | 19 | def __init__(self, state_path): 20 | # Variable to count total progress in the last days: 21 | # In case it is needed to see which relays are not being measured, 22 | # store their fingerprint, not only their number. 23 | self.measured_fp_set = set() 24 | self.consensus_fp_set = set() 25 | self.measured_percent = 0 26 | self.main_loop_tstart = time.monotonic() 27 | 28 | self.state_dict = State(state_path) 29 | 30 | self.previous_measurement_percent = 0 31 | 32 | def register_measured_fpr(self, async_result): 33 | self.measured_fp_set.add(async_result) 34 | 35 | def register_consensus_fprs(self, relay_fprs): 36 | for r in relay_fprs: 37 | self.consensus_fp_set.add(r) 38 | 39 | def print_heartbeat_message(self): 40 | """Print the new percentage of the different relays that were measured. 41 | 42 | This way it can be known whether the scanner is making progress 43 | measuring all the Network. 44 | 45 | Log the percentage, the number of relays measured and not measured, 46 | the number of loops and the time elapsed since it started measuring. 47 | """ 48 | loops_count = self.state_dict.count('recent_priority_list') 49 | 50 | not_measured_fp_set = self.consensus_fp_set.difference( 51 | self.measured_fp_set 52 | ) 53 | main_loop_tdelta = (time.monotonic() - self.main_loop_tstart) / 60 54 | new_measured_percent = round( 55 | len(self.measured_fp_set) / len(self.consensus_fp_set) * 100 56 | ) 57 | 58 | log.info("Run %s main loops.", loops_count) 59 | log.info("Measured in total %s (%s%%) unique relays in %s minutes", 60 | len(self.measured_fp_set), new_measured_percent, 61 | main_loop_tdelta) 62 | log.info("%s relays still not measured.", len(not_measured_fp_set)) 63 | 64 | # The case when it is equal will only happen when all the relays 65 | # have been measured. 66 | if (new_measured_percent <= self.previous_measurement_percent): 67 | log.warning("There is no progress measuring new unique relays.") 68 | 69 | self.previous_measurement_percent = new_measured_percent 70 | -------------------------------------------------------------------------------- /sbws/lib/scaling.py: -------------------------------------------------------------------------------- 1 | from statistics import mean 2 | 3 | from sbws.globals import RELAY_TYPES 4 | from sbws.util.stem import rs_relay_type 5 | 6 | 7 | def bw_measurements_from_results(results): 8 | return [ 9 | dl['amount'] / dl['duration'] 10 | for r in results for dl in r.downloads 11 | ] 12 | 13 | 14 | def bw_filt(bw_measurements): 15 | """Filtered bandwidth for a relay. 16 | 17 | It is the equivalent to Torflow's ``filt_sbw``. 18 | ``mu`` in this function is the equivalent to Torflow's ``sbw``. 19 | """ 20 | # It's safe to return 0 here, because: 21 | # 1. this value will be the numerator when calculating the ratio. 22 | # 2. `kb_round_x_sig_dig` returns a minimum of 1. 23 | # This should never be the case, as the measurements come from successful 24 | # results. 25 | if not bw_measurements: 26 | return 0 27 | # Torflow is rounding to an integer, so is `bw_mean_from_results` in 28 | # `v3bwfile.py` 29 | mu = round(mean(bw_measurements)) 30 | bws_gte_mean = list(filter(lambda bw: bw >= mu, bw_measurements)) 31 | if bws_gte_mean: 32 | return round(mean(bws_gte_mean)) 33 | return mu 34 | 35 | 36 | def network_means_by_relay_type(bw_lines, router_statuses_d): 37 | # Temporarily assign the type of relay to calculate network stream and 38 | # filtered bandwidth by type 39 | for line in bw_lines: 40 | rs = None 41 | if router_statuses_d: 42 | rs = router_statuses_d.get(line.node_id.replace("$", ""), None) 43 | line.set_relay_type(rs_relay_type(rs)) 44 | 45 | mu_type = muf_type = {} 46 | for rt in RELAY_TYPES: 47 | bw_lines_type = [line for line in bw_lines if line.relay_type == rt] 48 | if len(bw_lines_type) > 0: 49 | # Torflow does not round these values. 50 | # Ensure they won't be 0 to avoid division by 0 Exception 51 | mu_type[rt] = mean([line.bw_mean for line in bw_lines_type]) or 1 52 | muf_type[rt] = mean([line.bw_filt for line in bw_lines_type]) or 1 53 | return mu_type, muf_type 54 | -------------------------------------------------------------------------------- /sbws/sbws.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import sbws.core.cleanup 4 | import sbws.core.scanner 5 | import sbws.core.generate 6 | import sbws.core.stats 7 | from sbws.util.config import get_config 8 | from sbws.util.config import validate_config 9 | from sbws.util.config import configure_logging 10 | from sbws.util.parser import create_parser 11 | from sbws import __version__ as version 12 | from stem import __version__ as stem_version 13 | from requests.__version__ import __version__ as requests_version 14 | import platform 15 | import logging 16 | 17 | from sbws.util.fs import sbws_required_disk_space 18 | 19 | log = logging.getLogger(__name__) 20 | 21 | 22 | def _ensure_dirs(conf): 23 | log.debug('Ensuring all dirs exists.') 24 | # it is not needed to check sbws_home dir, since the following 25 | # will create parent dirs too (in case they don't exist) 26 | os.makedirs(conf.getpath('paths', 'datadir'), exist_ok=True) 27 | os.makedirs(conf.getpath('paths', 'v3bw_dname'), exist_ok=True) 28 | os.makedirs(conf.getpath('paths', 'log_dname'), exist_ok=True) 29 | 30 | 31 | def _adjust_log_level(args, conf): 32 | if not args.log_level: 33 | return 34 | conf['logger_sbws']['level'] = args.log_level 35 | 36 | 37 | def _get_startup_line(): 38 | py_ver = platform.python_version() 39 | py_plat = platform.platform() 40 | return 'sbws %s with python %s on %s, stem %s, and requests %s' % \ 41 | (version, py_ver, py_plat, stem_version, requests_version) 42 | 43 | 44 | def main(): 45 | parser = create_parser() 46 | args = parser.parse_args() 47 | conf = get_config(args) 48 | _ensure_dirs(conf) 49 | _adjust_log_level(args, conf) 50 | conf_valid, conf_errors = validate_config(conf) 51 | if not conf_valid: 52 | for e in conf_errors: 53 | log.critical(e) 54 | exit(1) 55 | configure_logging(args, conf) 56 | parser.description = sbws_required_disk_space(conf) 57 | def_args = [args, conf] 58 | def_kwargs = {} 59 | known_commands = { 60 | 'cleanup': {'f': sbws.core.cleanup.main, 61 | 'a': def_args, 'kw': def_kwargs}, 62 | 'scanner': {'f': sbws.core.scanner.main, 63 | 'a': def_args, 'kw': def_kwargs}, 64 | 'generate': {'f': sbws.core.generate.main, 65 | 'a': def_args, 'kw': def_kwargs}, 66 | 'stats': {'f': sbws.core.stats.main, 67 | 'a': def_args, 'kw': def_kwargs}, 68 | } 69 | try: 70 | if args.command not in known_commands: 71 | parser.print_help() 72 | else: 73 | log.info(_get_startup_line()) 74 | comm = known_commands[args.command] 75 | exit(comm['f'](*comm['a'], **comm['kw'])) 76 | except KeyboardInterrupt: 77 | print('') 78 | -------------------------------------------------------------------------------- /sbws/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torproject/sbws/fd139b89de20b6d38941f137325e05069995793b/sbws/util/__init__.py -------------------------------------------------------------------------------- /sbws/util/filelock.py: -------------------------------------------------------------------------------- 1 | import os 2 | import fcntl 3 | import logging 4 | from sbws.globals import fail_hard 5 | 6 | log = logging.getLogger(__name__) 7 | 8 | 9 | class _FLock: 10 | def __init__(self, lock_fname): 11 | self._lock_fname = lock_fname 12 | self._fd = None 13 | 14 | def __enter__(self): 15 | mode = os.O_RDWR | os.O_CREAT | os.O_TRUNC 16 | self._fd = os.open(self._lock_fname, mode) 17 | log.debug('Going to lock %s', self._lock_fname) 18 | try: 19 | fcntl.flock(self._fd, fcntl.LOCK_EX) 20 | except OSError as e: 21 | fail_hard('We couldn\'t call flock. Are you on an unsupported ' 22 | 'platform? Error: %s', e) 23 | log.debug('Received lock %s', self._lock_fname) 24 | 25 | def __exit__(self, exc_type, exc_val, exc_tb): 26 | if self._fd is not None: 27 | log.debug('Releasing lock %s', self._lock_fname) 28 | os.close(self._fd) 29 | 30 | 31 | class DirectoryLock(_FLock): 32 | ''' 33 | Holds a lock on a file in **dname** so that other sbws processes/threads 34 | won't try to read/write while we are reading/writing in this directory. 35 | 36 | >>> with DirectoryLock(dname): 37 | >>> # do some reading/writing in dname 38 | >>> # no longer have the lock 39 | 40 | Note: The directory must already exist. 41 | 42 | :param str dname: Name of directory for which we want to obtain a lock 43 | ''' 44 | def __init__(self, dname): 45 | assert os.path.isdir(dname) 46 | lock_fname = os.path.join(dname, '.lockfile') 47 | super().__init__(lock_fname) 48 | 49 | 50 | class FileLock(_FLock): 51 | ''' 52 | Holds a lock on **fname** so that other sbws processes/threads 53 | won't try to read/write while we are reading/writing this file. 54 | 55 | >>> with FileLock(fname): 56 | >>> # do some reading/writing of fname 57 | >>> # no longer have the lock 58 | 59 | :param str fname: Name of the file for which we want to obtain a lock 60 | ''' 61 | def __init__(self, fname): 62 | lock_fname = fname + '.lockfile' 63 | super().__init__(lock_fname) 64 | -------------------------------------------------------------------------------- /sbws/util/fs.py: -------------------------------------------------------------------------------- 1 | """Utils file system functions""" 2 | 3 | import logging 4 | import shutil 5 | 6 | log = logging.getLogger(__name__) 7 | 8 | DISK_SPACE_TEXT = """ 9 | Disk space requirements 10 | ----------------------- 11 | v3bw files: the maximum space required is ~{mb_bw} MB, after {d_bw} days. 12 | result files: the maximum space required is ~{mb_results} MB, after {d_r} days. 13 | tor directory: the space required is ~{mb_tor} MB. 14 | code and depenencies: the space required is ~{mb_code} MB 15 | Total disk space required is: ~{mb_total} MB 16 | """ 17 | 18 | 19 | def sbws_required_disk_space(conf): 20 | """Disk space required by sbws files. 21 | Rough calculations. 22 | 23 | :param ConfigParser conf: sbws configuration 24 | :returns: int, size in MB 25 | """ 26 | text_dict = {} 27 | # Number of relays per line average size in Bytes 28 | size_v3bw_file = 7500 * 220 29 | # default crontab configuration will run genenerate every hour 30 | num_v3bw_files_day = 24 31 | # ~1000 is the length of a line when the result is successfull 32 | # ~4550 is the number of lines of the biggest result file 33 | size_result_file = 4550 * 1000 34 | num_result_files_day = 1 35 | space_v3bw_files_day = size_v3bw_file * num_v3bw_files_day 36 | space_result_files_day = size_result_file * num_result_files_day 37 | size_compressed_files = 600 * 1024 38 | # default crontab configuration will run cleanup once a day 39 | # default cleanup configuration will compress v3bw files after 1 day 40 | # and delete them after 7 days 41 | v3bw_compress_after_days = conf.getint('cleanup', 42 | 'v3bw_files_compress_after_days') 43 | v3bw_delete_after_days = conf.getint('cleanup', 44 | 'v3bw_files_delete_after_days') 45 | v3bw_max_space_after_delete = \ 46 | (space_v3bw_files_day * v3bw_compress_after_days) + \ 47 | (size_compressed_files * num_v3bw_files_day * v3bw_delete_after_days) 48 | text_dict['mb_bw'] = round(v3bw_max_space_after_delete / 1000 ** 2) 49 | text_dict['d_bw'] = v3bw_delete_after_days 50 | # default crontab configuration will run cleanup once a day 51 | # default cleanup configuration will compress v3bw files after 1 day 52 | # and delete them after 7 days 53 | results_compress_after_days = conf.getint('cleanup', 54 | 'data_files_compress_after_days') 55 | results_delete_after_days = conf.getint('cleanup', 56 | 'data_files_delete_after_days') 57 | results_max_space_after_delete = \ 58 | (space_result_files_day * results_compress_after_days) + \ 59 | (size_compressed_files * num_v3bw_files_day * 60 | results_delete_after_days) 61 | text_dict['mb_results'] = round(results_max_space_after_delete / 1000 ** 2) 62 | text_dict['d_r'] = results_delete_after_days 63 | # not counted rotated files and assuming that when it is not rotated the 64 | # size will be aproximately 10MiB 65 | space_log_files = 0 66 | if conf.getboolean('logging', 'to_file'): 67 | size_log_file = conf.getint('logging', 'to_file_max_bytes') 68 | num_log_files = conf.getint('logging', 'to_file_num_backups') 69 | space_log_files = size_log_file * num_log_files 70 | text_dict['mb_log'] = space_log_files 71 | # roughly, size of a current tor dir 72 | size_tor_dir = 19828000 73 | text_dict['mb_tor'] = round(size_tor_dir / 1000 ** 2) 74 | # roughly, the size of this code and dependencies 75 | size_code_deps = 2097152 76 | text_dict['mb_code'] = round(size_code_deps / 1000 ** 2) 77 | # Multiply per 2, just in case 78 | size_total = (results_max_space_after_delete + 79 | v3bw_max_space_after_delete + space_log_files + 80 | size_tor_dir + size_code_deps) * 2 81 | text_dict['mb_total'] = round(size_total / 1000 ** 2) 82 | space_text = DISK_SPACE_TEXT.format(**text_dict) 83 | return space_text 84 | 85 | 86 | def df(path): 87 | # Not being used, since it makes a disk space system call and some 88 | # systems might not allow it 89 | """Return space left on device where path is in MiB.""" 90 | return round(shutil.disk_usage(path).free / (1024 ** 2)) 91 | 92 | 93 | def is_low_space(conf): 94 | # Not being used, since it makes a disk space system call and some 95 | # systems might not allow it 96 | """Warn and return True when the space left on the device is less than 97 | what is needed for sbws and False otherwise needs. 98 | """ 99 | disk_required_mb = sbws_required_disk_space(conf) 100 | disk_avail_mb = df(conf.getpath('paths', 'sbws_home')) 101 | if disk_avail_mb < disk_required_mb: 102 | log.warn("The space left on the device (%s MiB) is less than " 103 | "the minimum recommended to run sbws (%s MiB)." 104 | "Run sbws cleanup to delete old sbws generated files.", 105 | disk_avail_mb, disk_required_mb) 106 | return True 107 | return False 108 | -------------------------------------------------------------------------------- /sbws/util/iso3166.py: -------------------------------------------------------------------------------- 1 | """ 2 | ISO 3166 alpha-2 countries' codes. 3 | Obtained from https://en.wikipedia.org/wiki/List_of_ISO_3166_country_codes. 4 | Last updated 2019/02/05. 5 | ZZ is not the code of any country and it is used to denote any country, 6 | when the destination Web Server is in a CDN. 7 | """ 8 | # When the destination Web Server is in a CDN, the IP could be resolved by 9 | # the exit relay and obtain the country from the IP. 10 | 11 | # It would be better to use some standard location system for geopgraphic areas 12 | # that doesn't depend on political borders. 13 | # It should be possible to obtain IP address location in that system too. 14 | 15 | ISO_3166_ALPHA_2 = [ 16 | 'AF', 'AX', 'AL', 'DZ', 'AS', 'AD', 'AO', 'AI', 'AQ', 'AG', 'AR', 'AM', 17 | 'AW', 'AU', 'AT', 'AZ', 'BS', 'BH', 'BD', 'BB', 'BY', 'BE', 'BZ', 'BJ', 18 | 'BM', 'BT', 'BO', 'BQ', 'BA', 'BW', 'BV', 'BR', 'IO', 'BN', 'BG', 'BF', 19 | 'BI', 'CV', 'KH', 'CM', 'CA', 'KY', 'CF', 'TD', 'CL', 'CN', 'CX', 'CC', 20 | 'CO', 'KM', 'CD', 'CG', 'CK', 'CR', 'CI', 'HR', 'CU', 'CW', 'CY', 'CZ', 21 | 'DK', 'DJ', 'DM', 'DO', 'EC', 'EG', 'SV', 'GQ', 'ER', 'EE', 'SZ', 'ET', 22 | 'FK', 'FO', 'FJ', 'FI', 'FR', 'GF', 'PF', 'TF', 'GA', 'GM', 'GE', 'DE', 23 | 'GH', 'GI', 'GR', 'GL', 'GD', 'GP', 'GU', 'GT', 'GG', 'GN', 'GW', 'GY', 24 | 'HT', 'HM', 'VA', 'HN', 'HK', 'HU', 'IS', 'IN', 'ID', 'IR', 'IQ', 'IE', 25 | 'IM', 'IL', 'IT', 'JM', 'JP', 'JE', 'JO', 'KZ', 'KE', 'KI', 'KP', 'KR', 26 | 'KW', 'KG', 'LA', 'LV', 'LB', 'LS', 'LR', 'LY', 'LI', 'LT', 'LU', 'MO', 27 | 'MK', 'MG', 'MW', 'MY', 'MV', 'ML', 'MT', 'MH', 'MQ', 'MR', 'MU', 'YT', 28 | 'MX', 'FM', 'MD', 'MC', 'MN', 'ME', 'MS', 'MA', 'MZ', 'MM', 'NA', 'NR', 29 | 'NP', 'NL', 'NC', 'NZ', 'NI', 'NE', 'NG', 'NU', 'NF', 'MP', 'NO', 'OM', 30 | 'PK', 'PW', 'PS', 'PA', 'PG', 'PY', 'PE', 'PH', 'PN', 'PL', 'PT', 'PR', 31 | 'QA', 'RE', 'RO', 'RU', 'RW', 'BL', 'SH', 'KN', 'LC', 'MF', 'PM', 'VC', 32 | 'WS', 'SM', 'ST', 'SA', 'SN', 'RS', 'SC', 'SL', 'SG', 'SX', 'SK', 'SI', 33 | 'SB', 'SO', 'ZA', 'GS', 'SS', 'ES', 'LK', 'SD', 'SR', 'SJ', 'SE', 'CH', 34 | 'SY', 'TW', 'TJ', 'TZ', 'TH', 'TL', 'TG', 'TK', 'TO', 'TT', 'TN', 'TR', 35 | 'TM', 'TC', 'TV', 'UG', 'UA', 'AE', 'GB', 'UM', 'US', 'UY', 'UZ', 'VU', 36 | 'VE', 'VN', 'VG', 'VI', 'WF', 'EH', 'YE', 'ZM', 'ZW', 'ZZ' 37 | ] 38 | -------------------------------------------------------------------------------- /sbws/util/json.py: -------------------------------------------------------------------------------- 1 | """JSON custom serializers and deserializers.""" 2 | import datetime 3 | import json 4 | 5 | from .timestamps import DateTimeSeq, DateTimeIntSeq 6 | 7 | 8 | class CustomEncoder(json.JSONEncoder): 9 | """JSONEncoder that serializes datetime to ISO 8601 string.""" 10 | 11 | def default(self, obj): 12 | if isinstance(obj, DateTimeSeq) or isinstance(obj, DateTimeIntSeq): 13 | return [self.default(i) for i in obj.list()] 14 | if isinstance(obj, datetime.datetime): 15 | return obj.replace(microsecond=0).isoformat() 16 | else: 17 | return super().default(obj) 18 | 19 | 20 | class CustomDecoder(json.JSONDecoder): 21 | """JSONDecoder that deserializes ISO 8601 string to datetime.""" 22 | 23 | def decode(self, s, **kwargs): 24 | decoded = super().decode(s, **kwargs) 25 | return self.process(decoded) 26 | 27 | def process(self, obj): 28 | if isinstance(obj, list) and obj: 29 | return [self.process(item) for item in obj] 30 | if isinstance(obj, dict): 31 | return {key: self.process(value) for key, value in obj.items()} 32 | if isinstance(obj, str): 33 | try: 34 | return datetime.datetime.strptime(obj, "%Y-%m-%dT%H:%M:%S") 35 | except ValueError: 36 | try: 37 | datetime.datetime.strptime( 38 | obj, "%Y-%m-%dT%H:%M:%S.%f" 39 | ).replace(microsecond=0) 40 | except ValueError: 41 | pass 42 | except TypeError: 43 | pass 44 | return obj 45 | -------------------------------------------------------------------------------- /sbws/util/parser.py: -------------------------------------------------------------------------------- 1 | import sbws.core.cleanup 2 | import sbws.core.scanner 3 | import sbws.core.generate 4 | import sbws.core.stats 5 | from sbws import __version__ 6 | 7 | from argparse import ArgumentParser, RawTextHelpFormatter 8 | import os 9 | 10 | 11 | def _default_dot_sbws_dname(): 12 | home = os.path.expanduser('~') 13 | return os.path.join(home, '.sbws') 14 | 15 | 16 | def create_parser(): 17 | p = ArgumentParser(formatter_class=RawTextHelpFormatter) 18 | p.add_argument( 19 | '--version', action='version', help='sbws version', 20 | version='{}'.format(__version__)) 21 | p.add_argument('--log-level', 22 | choices=['debug', 'info', 'warning', 'error', 'critical'], 23 | help='Override the sbws log level') 24 | p.add_argument('-c', '--config', 25 | help='Path to the sbws config file') 26 | sub = p.add_subparsers(dest='command') 27 | sbws.core.cleanup.gen_parser(sub) 28 | sbws.core.scanner.gen_parser(sub) 29 | sbws.core.generate.gen_parser(sub) 30 | sbws.core.stats.gen_parser(sub) 31 | return p 32 | -------------------------------------------------------------------------------- /sbws/util/requests.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | from sbws import settings 4 | from sbws.util import stem as stem_utils 5 | 6 | 7 | class TimedSession(requests.Session): 8 | """Requests Session that sends timeout in the head and get methods. 9 | """ 10 | 11 | def get(self, url, **kwargs): 12 | return super().get(url, timeout=getattr(self, "_timeout", None), 13 | **kwargs) 14 | 15 | def head(self, url, **kwargs): 16 | return super().head(url, timeout=getattr(self, "_timeout", None), 17 | **kwargs) 18 | 19 | 20 | def make_session(controller, timeout): 21 | """ 22 | Initialize a TimedSession with the timeout, the proxies and the headers. 23 | 24 | """ 25 | s = TimedSession() 26 | socks_info = stem_utils.get_socks_info(controller) 27 | # Probably because scanner is stopping. 28 | if socks_info is None: 29 | return None 30 | s.proxies = { 31 | 'http': 'socks5h://{}:{}'.format(*socks_info), 32 | 'https': 'socks5h://{}:{}'.format(*socks_info), 33 | } 34 | # ``_timeout`` is not used by request's Session, but it is by TimedSession. 35 | s._timeout = timeout 36 | s.headers = settings.HTTP_HEADERS 37 | return s 38 | -------------------------------------------------------------------------------- /sbws/util/state.py: -------------------------------------------------------------------------------- 1 | from sbws.util.filelock import FileLock 2 | import os 3 | import json 4 | 5 | from .json import CustomDecoder, CustomEncoder 6 | 7 | 8 | class State: 9 | """ 10 | `json` wrapper to read a json file every time it gets a key and to write 11 | to the file every time a key is set. 12 | 13 | Every time a key is got or set, the file is locked, to atomically access 14 | and update the file across threads and across processes. 15 | 16 | >>> state = State('foo.state') 17 | >>> # state == {} 18 | 19 | >>> state['linux'] = True 20 | >>> # 'foo.state' now exists on disk with the JSON for {'linux': True} 21 | 22 | >>> # We read 'foo.state' from disk in order to get the most up-to-date 23 | >>> # state info. Pretend another process has updated 'linux' to be 24 | >>> # False 25 | >>> state['linux'] 26 | >>> # returns False 27 | 28 | >>> # Pretend another process has added the user's age to the state file. 29 | >>> # As before, we read the state file from disk for the most 30 | >>> # up-to-date info. 31 | >>> state['age'] 32 | >>> # Returns 14 33 | 34 | >>> # We now set their name. We read the state file first, set the option, 35 | >>> # and then write it out. 36 | >>> state['name'] = 'John' 37 | 38 | >>> # We can do many of the same things with a State object as with a dict 39 | >>> for key in state: print(key) 40 | >>> # Prints 'linux', 'age', and 'name' 41 | 42 | """ 43 | 44 | def __init__(self, fname): 45 | self._fname = fname 46 | self._state = self._read() 47 | 48 | def _read(self): 49 | if not os.path.exists(self._fname): 50 | return {} 51 | with FileLock(self._fname): 52 | with open(self._fname, 'rt') as fd: 53 | return json.load(fd, cls=CustomDecoder) 54 | 55 | def _write(self): 56 | with FileLock(self._fname): 57 | with open(self._fname, 'wt') as fd: 58 | return json.dump(self._state, fd, indent=4, cls=CustomEncoder) 59 | 60 | def __len__(self): 61 | self._state = self._read() 62 | return self._state.__len__() 63 | 64 | def get(self, key, d=None): 65 | """ 66 | Implements a dictionary ``get`` method reading and locking 67 | a json file. 68 | """ 69 | self._state = self._read() 70 | return self._state.get(key, d) 71 | 72 | def __getitem__(self, key): 73 | self._state = self._read() 74 | return self._state.__getitem__(key) 75 | 76 | def __delitem__(self, key): 77 | self._state = self._read() 78 | self._state.__delitem__(key) 79 | self._write() 80 | 81 | def __setitem__(self, key, value): 82 | # NOTE: important, read the file before setting the key, 83 | # otherwise if other instances are creating other keys, they're lost. 84 | self._state = self._read() 85 | self._state.__setitem__(key, value) 86 | self._write() 87 | 88 | def __iter__(self): 89 | self._state = self._read() 90 | return self._state.__iter__() 91 | 92 | def __contains__(self, item): 93 | self._state = self._read() 94 | return self._state.__contains__(item) 95 | 96 | def count(self, k): 97 | """ 98 | Returns the length if the key value is a list 99 | or the sum of number if the key value is a list of list 100 | or the key value 101 | or None if the state doesn't have the key. 102 | """ 103 | if self.get(k): 104 | if isinstance(self._state[k], list): 105 | if isinstance(self._state[k][0], list): 106 | return sum(map(lambda x: x[1], self._state[k])) 107 | return len(self._state[k]) 108 | return self.get(k) 109 | return None 110 | -------------------------------------------------------------------------------- /sbws/util/timestamp.py: -------------------------------------------------------------------------------- 1 | """Util functions to convert between timestamp formats""" 2 | from datetime import datetime, timedelta 3 | 4 | from ..globals import MEASUREMENTS_PERIOD 5 | 6 | 7 | def dt_obj_to_isodt_str(dt): 8 | """ 9 | Convert datetime object to ISO 8601 string. 10 | 11 | :param datetime dt: datetime object in UTC timezone 12 | :returns: ISO 8601 string 13 | """ 14 | assert isinstance(dt, datetime) 15 | # Using naive datetime object without timezone, assumed utc 16 | return dt.replace(microsecond=0).isoformat() 17 | 18 | 19 | def isostr_to_dt_obj(isostr): 20 | return datetime.strptime(isostr, "%Y-%m-%dT%H:%M:%S") 21 | 22 | 23 | def unixts_to_dt_obj(unixts): 24 | """ 25 | Convert unix timestamp to naive datetime object in UTC time zone. 26 | 27 | :param float/int/str unixts: unix timestamp 28 | :returns: datetime object in UTC timezone 29 | """ 30 | if isinstance(unixts, str): 31 | try: 32 | unixts = int(unixts) 33 | except ValueError: 34 | unixts = float(unixts) 35 | if isinstance(unixts, float): 36 | unixts = int(unixts) 37 | assert isinstance(unixts, int) 38 | return datetime.utcfromtimestamp(unixts) 39 | 40 | 41 | def unixts_to_isodt_str(unixts): 42 | """ 43 | Convert unix timestamp to ISO 8601 string in UTC time zone. 44 | 45 | :param float/int/str unixts: unix timestamp 46 | :returns: ISO 8601 string in UTC time zone 47 | """ 48 | return dt_obj_to_isodt_str(unixts_to_dt_obj(unixts)) 49 | 50 | 51 | def now_unixts(): 52 | return datetime.utcnow().timestamp() 53 | 54 | 55 | def now_isodt_str(): 56 | """Return datetime now as ISO 8601 string in UTC time zone.""" 57 | return dt_obj_to_isodt_str(datetime.utcnow()) 58 | 59 | 60 | def now_fname(): 61 | """ 62 | Return now timestamp in UTC formatted as %Y%m%d_%H%M%S string for file 63 | names. 64 | 65 | :returns: now timestamp in UTC formatted as %Y%m%d_%H%M%S string 66 | """ 67 | return datetime.utcnow().strftime("%Y%m%d_%H%M%S") 68 | 69 | 70 | def unixts_to_str(unixts): 71 | """Convert unix timestamp integer or float to string""" 72 | # even if it is only converting to str, ensure that input is nothing else 73 | # than int or float 74 | assert isinstance(unixts, int) or isinstance(unixts, float) 75 | return str(unixts) 76 | 77 | 78 | # XXX: tech-debt: replace all the code that check whether a 79 | # measurement or relay is older than the measurement period by this. 80 | def is_old(timestamp, measurements_period=MEASUREMENTS_PERIOD): 81 | """Whether the given timestamp is older that the given measurements 82 | period. 83 | """ 84 | if not isinstance(timestamp, datetime): 85 | if isinstance(timestamp, str): 86 | # This will raise an exception if the string is not correctly 87 | # formatted. 88 | timestamp = isostr_to_dt_obj(timestamp) 89 | elif isinstance(timestamp, int) or isinstance(timestamp, float): 90 | # This will raise an exception if the type is not int or float or 91 | # is not actually a timestamp 92 | timestamp = unixts_to_dt_obj(timestamp) 93 | oldest_date = datetime.utcnow() - timedelta(seconds=measurements_period) 94 | return timestamp < oldest_date 95 | -------------------------------------------------------------------------------- /sbws/util/timestamps.py: -------------------------------------------------------------------------------- 1 | """Util classes to manipulate sequences of datetime timestamps. 2 | 3 | Optionally update also a state file. 4 | 5 | """ 6 | # Workarounds to store datetimes for objects because they are not compossed 7 | # by other objects nor stored in a database with a creation datetime. 8 | import collections 9 | from datetime import datetime, timedelta 10 | import logging 11 | 12 | from sbws.util.timestamp import is_old 13 | 14 | log = logging.getLogger(__name__) 15 | 16 | 17 | class DateTimeSeq(collections.deque): 18 | """Store and manage a datetime sequence and optionally a state file.""" 19 | 20 | def __init__(self, iterable=[], maxlen=None, state=None, state_key=None): 21 | self._maxlen = maxlen 22 | self._items = collections.deque(iterable, maxlen) 23 | self._state = state 24 | self._state_key = state_key 25 | 26 | def _remove_old(self): 27 | self._items = collections.deque( 28 | filter(lambda x: not is_old(x), self._items), maxlen=self._maxlen 29 | ) 30 | 31 | def update(self, dt=None): 32 | self._remove_old() 33 | self._items.append(dt or datetime.utcnow().replace(microsecond=0)) 34 | if self._state is not None and self._state_key: 35 | self._state[self._state_key] = list(self._items) 36 | return list(self._items) 37 | 38 | def last(self): 39 | if len(self._items) > 0: 40 | return self._items[-1] 41 | return datetime.utcnow().replace(microsecond=0) - timedelta(hour=1) 42 | 43 | def list(self): 44 | return list(self._items) 45 | 46 | def __len__(self): 47 | return len(self._items) 48 | 49 | 50 | class DateTimeIntSeq(collections.deque): 51 | """ 52 | Store and manage a sequence of lists composed of a datetime and an int. 53 | 54 | Optionally store and manage an state file. 55 | """ 56 | 57 | def __init__(self, iterable=[], maxlen=None, state=None, state_key=None): 58 | self._maxlen = maxlen 59 | self._items = collections.deque(iterable, maxlen) 60 | self._state = state 61 | self._state_key = state_key 62 | 63 | def _remove_old(self): 64 | self._items = collections.deque( 65 | filter(lambda x: not is_old(x[0]), self._items), 66 | maxlen=self._maxlen, 67 | ) 68 | 69 | def update(self, dt=None, number=0): 70 | self._remove_old() 71 | # Because json serializes tuples to lists, use list instead of tuple 72 | # to facilitate comparisons. 73 | self._items.append( 74 | [dt or datetime.utcnow().replace(microsecond=0), number] 75 | ) 76 | if self._state is not None and self._state_key: 77 | self._state[self._state_key] = list(self._items) 78 | return list(self._items) 79 | 80 | def last(self): 81 | if len(self._items) > 0: 82 | return self._items[-1] 83 | return datetime.utcnow().replace(microsecond=0) - timedelta(hour=1) 84 | 85 | def list(self): 86 | return list(self._items) 87 | 88 | def __len__(self): 89 | return sum(map(lambda x: x[1], self._items)) 90 | -------------------------------------------------------------------------------- /sbws/util/userquery.py: -------------------------------------------------------------------------------- 1 | # Based on https://stackoverflow.com/a/3041990 2 | def query_yes_no(question, default='yes'): 3 | ''' 4 | Ask a yes/no question via input() and return the user's answer. 5 | 6 | :param str question: Prompt given to the user. 7 | :param str default: The assumed answer if th user just hits **Enter**. It 8 | must be ``'yes'`` (the default if no default is given), ``'no'``, or 9 | ``None`` (meaning an answer is required from the user). 10 | :returns: ``True`` if we ended up with a 'yes' answer, otherwise 11 | ``False``. 12 | ''' 13 | valid = {'yes': True, 'y': True, 'ye': True, 'no': False, 'n': False} 14 | if default is None: 15 | prompt = ' [y/n] ' 16 | elif default == 'yes': 17 | prompt = ' [Y/n] ' 18 | elif default == 'no': 19 | prompt = ' [y/N] ' 20 | else: 21 | raise ValueError('invalid default answer: "%s"' % default) 22 | prompt = question + prompt 23 | first_loop = True 24 | while True: 25 | choice = input(prompt).lower() 26 | if default is not None and choice == '': 27 | return valid[default] 28 | elif choice in valid: 29 | return valid[choice] 30 | elif first_loop: 31 | prompt = 'Please respond with "yes" or "no" (or y or n).\n' +\ 32 | prompt 33 | first_loop = False 34 | -------------------------------------------------------------------------------- /scripts/maint/update-authors: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Add <%aE> to the format string to get emails too 4 | AUTHORS=$(git log --format='%aN' | sort -u | 5 | while read LINE 6 | do 7 | printf "* $LINE\n" 8 | done | sort --ignore-case 9 | ) 10 | DATE=$(date +%Y-%m-%d) 11 | COMMIT=$(git log -n 1 --format='%h') 12 | 13 | cat << EOF 14 | # Authors 15 | 16 | The following people have contributed to Simple Bandwidth Scanner. 17 | Thank you for helping make Tor better. 18 | 19 | $AUTHORS 20 | 21 | *Last updated: $DATE on $COMMIT* 22 | EOF 23 | -------------------------------------------------------------------------------- /scripts/maint/update-website: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | function cleanup { 4 | echo -n '' 5 | } 6 | trap cleanup EXIT 7 | 8 | function fail_hard { 9 | echo "$1" 10 | exit 1 11 | } 12 | 13 | make=$(which gmake &>/dev/null && echo "gmake" || echo "make") 14 | sbws_repo="$(realpath $1)" 15 | venv="$sbws_repo/venv" 16 | 17 | [ -d "$venv" ] || fail_hard "$venv doesn't exist" 18 | [ ! -z "$sbws_repo" ] || fail_hard "$0 " 19 | [ -d "$sbws_repo" ] || fail_hard "$sbws_repo doesn't exist" 20 | 21 | pushd "$sbws_repo" 22 | 23 | git pull 24 | source "$venv/bin/activate" 25 | pip install -U --upgrade-strategy eager .[doc] 26 | 27 | pushd docs 28 | $make html 29 | popd 30 | -------------------------------------------------------------------------------- /scripts/tools/get-per-relay-budget.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # File: get-per-relay-budget.py 3 | # Written by: Matt Traudt 4 | # Copyright/License: CC0 5 | from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter 6 | from statistics import median 7 | from statistics import mean 8 | from stem.control import Controller 9 | import stem 10 | 11 | 12 | def _get_controller_port(args): 13 | return Controller.from_port(port=args.ctrl_port) 14 | 15 | 16 | def _get_controller_socket(args): 17 | return Controller.from_socket_file(path=args.ctrl_socket) 18 | 19 | 20 | def get_controller(args): 21 | try: 22 | cont = _get_controller_port(args) 23 | except stem.SocketError: 24 | cont = _get_controller_socket(args) 25 | return cont 26 | 27 | 28 | def print_quiet(bws): 29 | print(round(mean(bws))) 30 | 31 | 32 | def print_regular(bws): 33 | print(len(bws), 'relays') 34 | print('mean:', round(mean(bws))) 35 | print('median:', round(median(bws))) 36 | 37 | 38 | def main(args): 39 | cont = get_controller(args) 40 | cont.authenticate() 41 | bws = [ns.bandwidth for ns in cont.get_network_statuses()] 42 | if args.quiet: 43 | print_quiet(bws) 44 | else: 45 | print_regular(bws) 46 | 47 | 48 | def gen_parser(): 49 | d = 'Get the consensus weight for every relay in the current consensus '\ 50 | 'and print some information about them.' 51 | p = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter, 52 | description=d) 53 | p.add_argument('--ctrl-port', metavar='PORT', type=int, default=9051, 54 | help='Port on which to control the Tor client') 55 | p.add_argument('--ctrl-socket', metavar='SOCK', type=str, 56 | default='/var/run/tor/control', 57 | help='Path to socket on which to control the Tor client') 58 | p.add_argument('-q', '--quiet', action='store_true', 59 | help='If given, only print the mean bandwidth to stdout') 60 | return p 61 | 62 | 63 | if __name__ == '__main__': 64 | p = gen_parser() 65 | args = p.parse_args() 66 | exit(main(args)) 67 | -------------------------------------------------------------------------------- /scripts/tools/osx-extra-loopback.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # The test networks for sbws use IP addresses in the 127.10/16 space and OS X 3 | # only seems to give lo0 127.0.0.1/32. This adds 127.10.0.1-20 to lo0. 4 | for ((i=1;i<20;i++)) 5 | do 6 | sudo ifconfig lo0 alias 127.10.0.$i up 7 | done 8 | 9 | -------------------------------------------------------------------------------- /scripts/tools/sbws-http-server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # File: sbws-http-server.py 3 | # Author: Matt Traudt 4 | # License: CC0 5 | # 6 | # This script implements just enough of the HTTP protocol to work with Simple 7 | # Bandwidth Scanner. 8 | # 9 | # All requested URLs exist. All return 1 GiB of garbage data. We always speak 10 | # HTTP/1.1 because that's necessary for Keep-Alive request headers 11 | # (used by sbws scanners) to work. 12 | # 13 | # HEAD and GET requests are supported to the minimum extent necessary. 14 | # This essentially means that if the client sends Range request headers just 15 | # like sbws does, then we'll only send back the number of bytes they requested. 16 | # Indeed, this was the motivating reason for the complexity of this script; 17 | # normally I would have used SimpleHTTPRequestHandler unmodified. 18 | # 19 | # Don't breathe too hard or this script might break. 20 | from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter 21 | import http.server 22 | # import time 23 | 24 | FILE_SIZE = 1*1024*1024*1024 # 1 GiB 25 | 26 | 27 | def _get_resp_size_from_range(range_str): 28 | assert range_str.startswith('bytes=') 29 | range_str = range_str[len('bytes='):] 30 | start_byte, end_byte = range_str.split('-') 31 | return int(end_byte) - int(start_byte) + 1 32 | 33 | 34 | class MyHTTPRequestHandler(http.server.SimpleHTTPRequestHandler): 35 | protocol_version = 'HTTP/1.1' 36 | 37 | def __init__(self, *a, **kw): 38 | super().__init__(*a, **kw) 39 | 40 | def send_head(self, length): 41 | self.send_response(200) 42 | self.send_header('Content-Type', 'application/octet-stream') 43 | self.send_header('Content-Length', length) 44 | # self.send_header('Last-Modified', self.date_time_string(time.time())) 45 | self.end_headers() 46 | 47 | def do_GET(self): 48 | range_hdr = self.headers['Range'] 49 | if not range_hdr: 50 | num_bytes = FILE_SIZE 51 | else: 52 | assert range_hdr.startswith('bytes=') 53 | num_bytes = _get_resp_size_from_range(range_hdr) 54 | self.send_head(num_bytes) 55 | self.wfile.write(b'A' * num_bytes) 56 | 57 | def do_HEAD(self): 58 | self.send_head(FILE_SIZE) 59 | 60 | 61 | def main(args): 62 | addr = ('', args.port) 63 | print('Listening on', addr) 64 | httpd = http.server.HTTPServer(addr, MyHTTPRequestHandler) 65 | httpd.serve_forever() 66 | 67 | 68 | if __name__ == '__main__': 69 | parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) 70 | parser.add_argument( 71 | '-p', '--port', default=8000, type=int, help='Port on which to listen') 72 | args = parser.parse_args() 73 | try: 74 | exit(main(args)) 75 | except KeyboardInterrupt: 76 | pass 77 | -------------------------------------------------------------------------------- /scripts/tools/scale-v3bw-with-budget.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # File: scale-v3bw-with-budget.py 3 | # Written by: Matt Traudt 4 | # Copyright/License: CC0 5 | from collections import OrderedDict 6 | from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, FileType 7 | import sys 8 | 9 | 10 | def fail_hard(*s): 11 | print(*s, file=sys.stderr) 12 | exit(1) 13 | 14 | 15 | def line_into_dict(line): 16 | words = line.strip().split() 17 | d = OrderedDict() 18 | for word in words: 19 | key, value = word.split('=') 20 | d.update({key: value}) 21 | return d 22 | 23 | 24 | def main(args): 25 | total_input_weight = 0 26 | line_dicts = [] 27 | for line in args.input: 28 | if 'node_id=' not in line: 29 | args.output.write(line) 30 | continue 31 | d = line_into_dict(line) 32 | # Check that the required parts of the line are here 33 | if 'node_id' not in d: 34 | fail_hard('Line without required node_id:', line) 35 | if 'bw' not in d: 36 | fail_hard('Line without required bw:', line) 37 | # Make sure the bw looks like an int 38 | try: 39 | d['bw'] = int(d['bw']) 40 | except ValueError as e: 41 | fail_hard('Found a non-int bw value', d['bw'], e) 42 | # Accumulate the total "bandwidth" weights on the input side 43 | total_input_weight += d['bw'] 44 | # And keep the line for later 45 | line_dicts.append(d) 46 | # Now calculate a ratio to multiply every line by. It's the total budget we 47 | # should give ourselves based on the number of relays in the v3bw file (AKA 48 | # the total output weight) divided by the total input weight 49 | ratio = (len(line_dicts) * args.budget_per_relay) / total_input_weight 50 | for d in line_dicts: 51 | d['bw'] = round(d['bw'] * ratio) 52 | # Accumulate all the parts of the line back together 53 | s = '' 54 | for key in d: 55 | s += '{}={} '.format(key, d[key]) 56 | # Remove trailing ' ' and replace with '\n' 57 | s = s.rstrip() + '\n' 58 | args.output.write(s) 59 | 60 | 61 | def gen_parser(): 62 | d = 'Read a v3bw file, adjust the bandwidth weights of the relays, and '\ 63 | 'write the new v3bw file out. For each relay in the v3bw file, we '\ 64 | 'give ourselves some amount of weight to work with. We then '\ 65 | 'distribute this weight to the relays in the same proportions their '\ 66 | 'input weights were in. This cases the scale of theie weights to '\ 67 | 'move up or down, but their relative weights stay the same.' 68 | p = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter, 69 | description=d) 70 | p.add_argument('-i', '--input', type=FileType('rt'), 71 | default='/dev/stdin', 72 | help='Input v3bw file to be scaled') 73 | p.add_argument('-o', '--output', type=FileType('wt'), 74 | default='/dev/stdout', 75 | help='Where to write a new, and scaled, v3bw file') 76 | p.add_argument('--budget-per-relay', type=float, default=7500, 77 | help='Per relay in the v3bw file, add this much to our ' 78 | 'budget') 79 | return p 80 | 81 | 82 | if __name__ == '__main__': 83 | p = gen_parser() 84 | args = p.parse_args() 85 | exit(main(args)) 86 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | # See the docstring in versioneer.py for instructions. Note that you must 2 | # re-run 'versioneer.py setup' after changing this section, and commit the 3 | # resulting files. 4 | [versioneer] 5 | VCS = git 6 | # Will generate versions in the form TAG[+DISTANCE.gSHORTHASH[.dirty]] , using 7 | # information from git describe --tags --dirty --always. 8 | style = pep440 9 | # A project-relative pathname into which the generated version strings should 10 | # be written 11 | versionfile_source = sbws/_version.py 12 | # As versionfile_source, relative to the build directory. 13 | versionfile_build = sbws/_version.py 14 | # Strimg at the start of all VCS tags. 15 | tag_prefix = v 16 | # String at the start of all unpacked tarball filenames. 17 | parentdir_prefix = sbws- 18 | 19 | [tool:pytest] 20 | log_cli=true 21 | log_cli_level=DEBUG 22 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Always prefer setuptools over distutils 3 | from setuptools import setup, find_packages 4 | # To use a consistent encoding 5 | from codecs import open 6 | import os 7 | # To generate the version at build time based on 8 | # git describe --tags --dirty --always 9 | import versioneer 10 | 11 | 12 | here = os.path.abspath(os.path.dirname(__file__)) 13 | 14 | 15 | def long_description(): 16 | with open(os.path.join(here, 'README.md'), encoding='utf-8') as f: 17 | return f.read() 18 | 19 | 20 | def get_package_data(): 21 | # Example that grabs all *.ini files in the cwd and all files in foo/bar 22 | # other_files = ['*.ini'] 23 | # for r, _, fs in os.walk(os.path.join(here, 'foo', 'bar')): 24 | # for f in fs: 25 | # other_files.append(os.path.join(r, f)) 26 | # return other_files 27 | return [ 28 | 'config.default.ini', 29 | 'config.log.default.ini', 30 | ] 31 | 32 | 33 | def get_data_files(): 34 | pass 35 | 36 | 37 | setup( 38 | name='sbws', 39 | version=versioneer.get_version(), 40 | cmdclass=versioneer.get_cmdclass(), 41 | description='Simple Bandwidth Scanner', 42 | long_description=long_description(), 43 | long_description_content_type="text/markdown", 44 | author='Matt Traudt, juga', 45 | author_email='{pastly, juga}@torproject.org', 46 | license='CC0', 47 | url="https://gitweb.torproject.org/sbws.git", 48 | classifiers=[ 49 | 'Development Status :: 4 - Beta', 50 | "Environment :: Console", 51 | 'Intended Audience :: Developers', 52 | 'Intended Audience :: System Administrators', 53 | 'Operating System :: OS Independent', 54 | 'Programming Language :: Python :: 3', 55 | 'Programming Language :: Python :: 3.6', 56 | 'Programming Language :: Python :: 3.7', 57 | 'Programming Language :: Python :: 3.8', 58 | 'Programming Language :: Python :: 3.9', 59 | 'Topic :: System :: Networking', 60 | ], 61 | packages=find_packages(), 62 | include_package_data=True, 63 | package_data={ 64 | 'sbws': get_package_data(), 65 | }, 66 | data_files=get_data_files(), 67 | keywords='tor onion bandwidth measurements scanner relay circuit', 68 | python_requires='>=3.5', 69 | entry_points={ 70 | 'console_scripts': [ 71 | 'sbws = sbws.sbws:main', 72 | ] 73 | }, 74 | install_requires=[ 75 | 'stem>=1.7.0', 76 | 'requests[socks]', 77 | ], 78 | extras_require={ 79 | # vulture: find unused code 80 | 'dev': ['flake8', 'vulture'], 81 | 'test': ['tox', 'pytest', 'coverage', 'freezegun'], 82 | # recommonmark: to make sphinx render markdown 83 | 'doc': ['sphinx', 'recommonmark', 'pylint'], 84 | }, 85 | ) 86 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torproject/sbws/fd139b89de20b6d38941f137325e05069995793b/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | """Common pytest configuration for unit and integration tests.""" 2 | import pytest 3 | import os.path 4 | from unittest import mock 5 | 6 | from freezegun import freeze_time 7 | from stem import descriptor 8 | 9 | from sbws import settings 10 | from sbws.lib import relaylist 11 | from sbws.lib import relayprioritizer 12 | from sbws.lib import resultdump 13 | from sbws.util.parser import create_parser 14 | 15 | 16 | @pytest.fixture(scope='session') 17 | def parser(): 18 | return create_parser() 19 | 20 | 21 | @pytest.fixture() 22 | def datadir(request): 23 | """get, read, open test files from the tests relative "data" directory.""" 24 | class D: 25 | def __init__(self, basepath): 26 | self.basepath = basepath 27 | 28 | def open(self, name, mode="r"): 29 | return self.basepath.join(name).open(mode) 30 | 31 | def join(self, name): 32 | return self.basepath.join(name).strpath 33 | 34 | def read(self, name): 35 | with self.open(name, "r") as f: 36 | return f.read() 37 | 38 | def readlines(self, name): 39 | with self.open(name, "r") as f: 40 | return f.readlines() 41 | return D(request.fspath.dirpath("data")) 42 | 43 | 44 | @pytest.fixture(scope="session") 45 | def root_data_path(): 46 | """Path to the data dir in the tests root, for both unit and integration 47 | tests. 48 | """ 49 | return os.path.join(os.path.dirname(os.path.abspath(__file__)), "data",) 50 | 51 | 52 | @pytest.fixture(scope="session") 53 | def router_statuses(root_data_path): 54 | p = os.path.join(root_data_path, "2020-02-29-10-00-00-consensus") 55 | network_statuses = descriptor.parse_file(p) 56 | network_statuses_list = list(network_statuses) 57 | return network_statuses_list 58 | 59 | 60 | @pytest.fixture(scope="session") 61 | def router_statuses_1h_later(root_data_path): 62 | p = os.path.join(root_data_path, "2020-02-29-11-00-00-consensus") 63 | network_statuses = descriptor.parse_file(p) 64 | network_statuses_list = list(network_statuses) 65 | return network_statuses_list 66 | 67 | 68 | @pytest.fixture(scope="session") 69 | def router_statuses_5days_later(root_data_path): 70 | p = os.path.join(root_data_path, "2020-03-05-10-00-00-consensus") 71 | network_statuses = descriptor.parse_file(p) 72 | network_statuses_list = list(network_statuses) 73 | return network_statuses_list 74 | 75 | 76 | @pytest.fixture(scope="session") 77 | def controller(router_statuses): 78 | controller = mock.Mock() 79 | controller.get_network_statuses.return_value = router_statuses 80 | return controller 81 | 82 | 83 | @pytest.fixture(scope="session") 84 | def controller_1h_later(router_statuses_1h_later): 85 | controller = mock.Mock() 86 | controller.get_network_statuses.return_value = router_statuses_1h_later 87 | return controller 88 | 89 | 90 | @pytest.fixture(scope="session") 91 | def controller_5days_later(router_statuses_5days_later): 92 | controller = mock.Mock() 93 | controller.get_network_statuses.return_value = router_statuses_5days_later 94 | return controller 95 | 96 | 97 | @pytest.fixture(scope="session") 98 | def server_descriptors(root_data_path): 99 | p = os.path.join(root_data_path, "2020-02-29-10-05-00-server-descriptors") 100 | server_descriptors = descriptor.parse_file(p) 101 | server_descriptors_list = list(server_descriptors) 102 | return server_descriptors_list 103 | 104 | 105 | @pytest.fixture(scope="session") 106 | def server_descriptor(server_descriptors): 107 | return server_descriptors[0] 108 | 109 | 110 | @pytest.fixture(scope="session") 111 | def router_status(server_descriptor, router_statuses): 112 | rs = [ 113 | ns 114 | for ns in router_statuses 115 | if ns.fingerprint == server_descriptor.fingerprint 116 | ][0] 117 | return rs 118 | 119 | 120 | # Because of the function scoped `args` in `tests.unit.conftest`, this has to 121 | # be function scoped too. 122 | @pytest.fixture(scope='function') 123 | def relay_list(args, conf, controller): 124 | """Returns a RelayList containing the Relays in the controller""" 125 | with freeze_time("2020-02-29 10:00:00"): 126 | return relaylist.RelayList(args, conf, controller) 127 | 128 | 129 | @pytest.fixture(scope='function') 130 | def result_dump(args, conf): 131 | """Returns a ResultDump without Results""" 132 | # To stop the thread that would be waiting for new results 133 | settings.set_end_event() 134 | return resultdump.ResultDump(args, conf) 135 | 136 | 137 | @pytest.fixture(scope="function") 138 | def relay_prioritizer(args, conf_results, relay_list, result_dump): 139 | """ 140 | Returns a RelayPrioritizer with a RelayList and a ResultDump. 141 | """ 142 | return relayprioritizer.RelayPrioritizer( 143 | args, conf_results, relay_list, result_dump 144 | ) 145 | -------------------------------------------------------------------------------- /tests/data/.sbws/state.dat: -------------------------------------------------------------------------------- 1 | { 2 | "scanner_started": "2020-02-29T10:00:00", 3 | "uuid": "e4ecf294-f253-478c-8660-28cbdfc690de", 4 | "tor_version": "0.4.2.6", 5 | "recent_consensus": [ 6 | "2020-03-18T13:26:46" 7 | ], 8 | "recent_priority_list": [ 9 | "2020-03-18T13:26:46" 10 | ], 11 | "recent_priority_relay": [ 12 | [ 13 | "2020-03-18T13:26:46", 14 | 15 15 | ] 16 | ], 17 | "recent_measurement_attempt": [ 18 | "2020-03-18T13:26:46", 19 | "2020-03-18T13:26:46", 20 | "2020-03-18T13:26:46", 21 | "2020-03-18T13:26:46", 22 | "2020-03-18T13:26:46", 23 | "2020-03-18T13:26:46", 24 | "2020-03-18T13:26:46", 25 | "2020-03-18T13:26:46", 26 | "2020-03-18T13:26:46", 27 | "2020-03-18T13:26:46", 28 | "2020-03-18T13:26:46", 29 | "2020-03-18T13:26:46", 30 | "2020-03-18T13:26:46", 31 | "2020-03-18T13:26:46", 32 | "2020-03-18T13:26:46" 33 | ], 34 | "min_perc_reached": null 35 | } 36 | -------------------------------------------------------------------------------- /tests/integration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torproject/sbws/fd139b89de20b6d38941f137325e05069995793b/tests/integration/__init__.py -------------------------------------------------------------------------------- /tests/integration/chutney_data/bwscanner: -------------------------------------------------------------------------------- 1 | # By default, Authorities are not configured as exits 2 | Authority = Node(tag="a", authority=1, relay=1, torrc="authority.tmpl") 3 | NonExitRelay = Node(tag="m", relay=1, exit=0, torrc="non-exit.tmpl") 4 | ExitRelay = Node(tag="r", relay=1, exit=1, torrc="relay.tmpl") 5 | Client = Node(tag="c", client=1, torrc="client_bwscanner.tmpl") 6 | RelayMAB = Node(tag="relay1mbyteMAB", relay=1, torrc="relay-MAB.tmpl") 7 | RelayMBR = Node(tag="relay1mbyteMBR", relay=1, torrc="relay-MBR.tmpl") 8 | 9 | NODES = Authority.getN(3) + \ 10 | NonExitRelay.getN(9) + \ 11 | ExitRelay.getN(3) + Client.getN(1) 12 | 13 | # RelayMBR.getN(1) + RelayMAB.getN(1) + \ 14 | 15 | ConfigureNodes(NODES) 16 | -------------------------------------------------------------------------------- /tests/integration/chutney_data/client_bwscanner.tmpl: -------------------------------------------------------------------------------- 1 | ${include:common.i} 2 | SocksPort $socksport 3 | 4 | #NOTE: Setting TestingClientConsensusDownloadSchedule doesn't 5 | # help -- dl_stats.schedule is not DL_SCHED_CONSENSUS 6 | # at boostrap time. 7 | # Try to download after: 8 | # the minimum initial consensus time to start with, 9 | # a few eager fetches, 10 | # then half the minimum testing consensus interval 11 | #TestingClientDownloadSchedule 0, 5 12 | #TestingClientConsensusDownloadSchedule 0, 5 13 | #ControlPort 8015 14 | UseEntryGuards 0 15 | UseMicroDescriptors 0 16 | FetchDirInfoEarly 1 17 | FetchDirInfoExtraEarly 1 18 | FetchUselessDescriptors 1 19 | LearnCircuitBuildTimeout 0 20 | CircuitBuildTimeout 60 21 | ConnectionPadding 0 22 | __DisablePredictedCircuits 1 23 | __LeaveStreamsUnattached 1 24 | -------------------------------------------------------------------------------- /tests/integration/chutney_data/non-exit.tmpl: -------------------------------------------------------------------------------- 1 | ${include:relay-non-exit.tmpl} 2 | 3 | ExitRelay 0 4 | ExitPolicy reject *:* 5 | -------------------------------------------------------------------------------- /tests/integration/chutney_data/relay-MAB.tmpl: -------------------------------------------------------------------------------- 1 | ${include:non-exit.tmpl} 2 | 3 | Nickname relay1mbyteMAB 4 | MaxAdvertisedBandwidth 1 MBytes 5 | -------------------------------------------------------------------------------- /tests/integration/chutney_data/relay-MBR.tmpl: -------------------------------------------------------------------------------- 1 | ${include:non-exit.tmpl} 2 | 3 | Nickname relay1mbyteMBR 4 | RelayBandwidthRate 1 MBytes 5 | -------------------------------------------------------------------------------- /tests/integration/conftest.py: -------------------------------------------------------------------------------- 1 | """pytest configuration for integration tests.""" 2 | import argparse 3 | import pytest 4 | import os 5 | 6 | from sbws.lib.circuitbuilder import GapsCircuitBuilder as CB 7 | from sbws.lib.destination import DestinationList 8 | from sbws.lib.relaylist import RelayList 9 | from sbws.util.config import _get_default_config 10 | from sbws.util.stem import launch_or_connect_to_tor 11 | 12 | 13 | class _PseudoArguments(argparse.Namespace): 14 | 15 | """Just enough of the argparse.Namespace (what you get when you do 16 | args = parser.parse_args()) to make get_config() happy 17 | 18 | >>> args = _PseudoArguments(directory='/home/matt/.sbws') 19 | >>> args.directory 20 | '/home/matt/.sbws' 21 | 22 | """ 23 | 24 | def __init__(self, **kw): 25 | for key in kw: 26 | setattr(self, key, kw[key]) 27 | 28 | 29 | @pytest.fixture(scope='session') 30 | def tmpdir(tmpdir_factory, request): 31 | """Create a tmp dir for the tests""" 32 | base = str(hash(request.node.nodeid))[:3] 33 | bn = tmpdir_factory.mktemp(base) 34 | return bn 35 | 36 | 37 | @pytest.fixture(scope='session') 38 | def sbwshome_empty(tmpdir): 39 | """Create sbws home inside of the test net tmp dir without initializing.""" 40 | home = "/tmp/.sbws" 41 | os.makedirs(home, exist_ok=True) 42 | return home 43 | 44 | 45 | @pytest.fixture(scope='session') 46 | def sbwshome_dir(sbwshome_empty): 47 | """Create sbws home inside of the test net tmp dir without initializing.""" 48 | os.makedirs(os.path.join(sbwshome_empty, 'datadir'), exist_ok=True) 49 | return sbwshome_empty 50 | 51 | 52 | @pytest.fixture(scope='session') 53 | def test_config_path(tmpdir): 54 | """""" 55 | config = tmpdir.join('.sbws.ini') 56 | return config 57 | 58 | 59 | @pytest.fixture(scope='session') 60 | def args(sbwshome_empty, parser, test_config_path): 61 | """Args with sbws home in the tests tmp dir.""" 62 | args = _PseudoArguments(config=test_config_path, output=sbwshome_empty, 63 | scale=False, log_level='debug') 64 | return args 65 | 66 | 67 | @pytest.fixture(scope='session') 68 | def conf(sbwshome_dir): 69 | """Default configuration with sbws home in the tmp test dir.""" 70 | conf = _get_default_config() 71 | conf['paths']['sbws_home'] = sbwshome_dir 72 | conf["paths"]["state_fpath"] = os.path.join(sbwshome_dir, "state.dat") 73 | conf['tor']['run_dpath'] = os.path.join(sbwshome_dir, 'tor', 'run') 74 | conf['destinations']['foo'] = 'on' 75 | conf['destinations.foo'] = {} 76 | # The test server is not using TLS. Ideally it should also support TLS 77 | # If the url would start with https but the request is not using TLS, 78 | # the request would hang. 79 | conf['destinations.foo']['url'] = 'http://127.0.0.1:28888/sbws.bin' 80 | conf['tor']['external_control_port'] = '8015' 81 | return conf 82 | 83 | 84 | @pytest.fixture(scope='session') 85 | def persistent_launch_tor(conf): 86 | cont = launch_or_connect_to_tor(conf) 87 | return cont 88 | 89 | 90 | @pytest.fixture(scope='session') 91 | def rl(args, conf, persistent_launch_tor): 92 | return RelayList(args, conf, persistent_launch_tor) 93 | 94 | 95 | @pytest.fixture(scope='session') 96 | def cb(args, conf, persistent_launch_tor, rl): 97 | return CB(args, conf, persistent_launch_tor, rl) 98 | 99 | 100 | @pytest.fixture(scope='session') 101 | def dests(args, conf, persistent_launch_tor, cb, rl): 102 | dests, error_msg = DestinationList.from_config(conf, cb, rl, 103 | persistent_launch_tor) 104 | assert dests, error_msg 105 | return dests 106 | 107 | 108 | # @pytest.fixture(scope='session') 109 | -------------------------------------------------------------------------------- /tests/integration/core/test_scanner.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from sbws.core.scanner import measure_relay 4 | from sbws.lib.resultdump import ResultSuccess 5 | import logging 6 | 7 | 8 | def assert_within(value, target, radius): 9 | ''' 10 | Assert that **value** is within **radius** of **target** 11 | 12 | If target is 10 and radius is 2, value can be anywhere between 8 and 12 13 | inclusive 14 | ''' 15 | assert target - radius < value, 'Value is too small. {} is not within '\ 16 | '{} of {}'.format(value, radius, target) 17 | assert target + radius > value, 'Value is too big. {} is not within '\ 18 | '{} of {}'.format(value, radius, target) 19 | 20 | 21 | @pytest.mark.skip(reason=("Disabled because chutney is not creating a network" 22 | "with relay1mbyteMAB.")) 23 | def test_measure_relay_with_maxadvertisedbandwidth( 24 | persistent_launch_tor, sbwshome_dir, args, conf, 25 | dests, cb, rl, caplog): 26 | caplog.set_level(logging.DEBUG) 27 | # d = get_everything_to_measure(sbwshome, cont, args, conf) 28 | # rl = d['rl'] 29 | # dests = d['dests'] 30 | # cb = d['cb'] 31 | # 117A456C911114076BEB4E757AC48B16CC0CCC5F is relay1mbyteMAB 32 | relay = [r for r in rl.relays 33 | if r.nickname == 'relay1mbyteMAB'][0] 34 | # d['relay'] = relay 35 | result = measure_relay(args, conf, dests, cb, rl, relay) 36 | assert len(result) == 1 37 | result = result[0] 38 | assert isinstance(result, ResultSuccess) 39 | one_mbyte = 1 * 1024 * 1024 40 | dls = result.downloads 41 | for dl in dls: 42 | # This relay has MaxAdvertisedBandwidth set, but should not be limited 43 | # to just 1 Mbyte. Assume and assert that all downloads where at least 44 | # more than 10% faster than 1 MBps 45 | assert dl['amount'] / dl['duration'] > one_mbyte * 1.1 46 | assert result.relay_average_bandwidth == one_mbyte 47 | 48 | 49 | @pytest.mark.skip(reason="temporally disabled") 50 | def test_measure_relay_with_relaybandwidthrate( 51 | persistent_launch_tor, args, conf, dests, cb, rl): 52 | relay = [r for r in rl.relays 53 | if r.nickname == 'relay1mbyteRBR'][0] 54 | result = measure_relay(args, conf, dests, cb, rl, relay) 55 | assert len(result) == 1 56 | result = result[0] 57 | assert isinstance(result, ResultSuccess) 58 | one_mbyte = 1 * 1024 * 1024 59 | allowed_error = 0.1 * one_mbyte # allow 10% error in either direction 60 | dls = result.downloads 61 | for dl in dls: 62 | assert_within(dl['amount'] / dl['duration'], one_mbyte, allowed_error) 63 | -------------------------------------------------------------------------------- /tests/integration/lib/test_circuitbuilder.py: -------------------------------------------------------------------------------- 1 | """Integration tests for circutibuilder.py""" 2 | import random 3 | 4 | 5 | def test_build_circuit(cb, rl): 6 | # Path is empty 7 | path = [] 8 | circuit_id, _ = cb.build_circuit(path) 9 | assert not circuit_id 10 | # Valid path, not valid exit 11 | exits = rl.exits_not_bad_allowing_port(port=443) 12 | # See https://gitlab.torproject.org/tpo/core/chutney/-/issues/40013: 13 | # Work around to get supposed non-exits because chutney is putting Exit 14 | # flag to all relays 15 | non_exits = list(set(rl.exits).difference(set(exits))) 16 | entry = random.choice(non_exits) 17 | # Because in chutney all relays are exits, we can't test using a non-exit 18 | # as 2nd hop. 19 | # Valid path and relays 20 | exit_relay = random.choice(exits) 21 | path = [entry.fingerprint, exit_relay.fingerprint] 22 | circuit_id, _ = cb.build_circuit(path) 23 | assert circuit_id 24 | -------------------------------------------------------------------------------- /tests/integration/lib/test_destination.py: -------------------------------------------------------------------------------- 1 | """Integration tests for destination.py""" 2 | import sbws.util.requests as requests_utils 3 | from sbws.lib.destination import (DestinationList, Destination, 4 | connect_to_destination_over_circuit) 5 | 6 | 7 | def test_destination_list_no_usability_test_success( 8 | conf, persistent_launch_tor, cb, rl 9 | ): 10 | # In a future refactor, if DestionationList is not initialized with the 11 | # controller, this test should be an unit test. 12 | destination_list, error_msg = DestinationList.from_config( 13 | conf, cb, rl, persistent_launch_tor 14 | ) 15 | # Because there's only 1 destination in conftest, random should return 16 | # the same one. 17 | assert destination_list.next() == \ 18 | destination_list._all_dests[0] 19 | 20 | 21 | def test_connect_to_destination_over_circuit_success(persistent_launch_tor, 22 | dests, cb, rl): 23 | destination = dests.next() 24 | session = requests_utils.make_session(persistent_launch_tor, 10) 25 | # Choose a relay that is not an exit 26 | relay = [r for r in rl.relays 27 | if r.nickname == 'test005m'][0] 28 | # Choose an exit, for this test it does not matter the bandwidth 29 | helper = rl.exits_not_bad_allowing_port(destination.port)[0] 30 | circuit_path = [relay.fingerprint, helper.fingerprint] 31 | # build a circuit 32 | circuit_id, _ = cb.build_circuit(circuit_path) 33 | # Perform "usability test" 34 | is_usable, response = connect_to_destination_over_circuit( 35 | destination, circuit_id, session, persistent_launch_tor, 1024) 36 | assert is_usable is True 37 | assert 'content_length' in response 38 | assert destination.is_functional() 39 | 40 | 41 | def test_connect_to_destination_over_circuit_fail(persistent_launch_tor, 42 | dests, cb, rl): 43 | bad_destination = Destination('https://example.example', 1024, False) 44 | session = requests_utils.make_session(persistent_launch_tor, 10) 45 | # Choose a relay that is not an exit 46 | relay = [r for r in rl.relays 47 | if r.nickname == 'test005m'][0] 48 | # Choose an exit, for this test it does not matter the bandwidth 49 | helper = rl.exits_not_bad_allowing_port(bad_destination.port)[0] 50 | circuit_path = [relay.fingerprint, helper.fingerprint] 51 | # Build a circuit. 52 | circuit_id, _ = cb.build_circuit(circuit_path) 53 | # Perform "usability test" 54 | is_usable, response = connect_to_destination_over_circuit( 55 | bad_destination, circuit_id, session, persistent_launch_tor, 1024) 56 | assert is_usable is False 57 | 58 | # because it is the first time it fails, failures aren't count 59 | assert bad_destination.is_functional() 60 | 61 | # fail three times in a row 62 | is_usable, response = connect_to_destination_over_circuit( 63 | bad_destination, circuit_id, session, persistent_launch_tor, 1024) 64 | is_usable, response = connect_to_destination_over_circuit( 65 | bad_destination, circuit_id, session, persistent_launch_tor, 1024) 66 | assert not bad_destination.is_functional() 67 | 68 | 69 | def test_functional_destinations(conf, cb, rl, persistent_launch_tor): 70 | good_destination = Destination('https://127.0.0.1:28888', 1024, False) 71 | bad_destination = Destination('https://example.example', 1024, False) 72 | 73 | session = requests_utils.make_session(persistent_launch_tor, 10) 74 | # Choose a relay that is not an exit 75 | relay = [r for r in rl.relays 76 | if r.nickname == 'test005m'][0] 77 | # Choose an exit, for this test it does not matter the bandwidth 78 | helper = rl.exits_not_bad_allowing_port(bad_destination.port)[0] 79 | circuit_path = [relay.fingerprint, helper.fingerprint] 80 | # Build a circuit. 81 | circuit_id, _ = cb.build_circuit(circuit_path) 82 | 83 | # fail three times in a row 84 | is_usable, response = connect_to_destination_over_circuit( 85 | bad_destination, circuit_id, session, persistent_launch_tor, 1024) 86 | is_usable, response = connect_to_destination_over_circuit( 87 | bad_destination, circuit_id, session, persistent_launch_tor, 1024) 88 | is_usable, response = connect_to_destination_over_circuit( 89 | bad_destination, circuit_id, session, persistent_launch_tor, 1024) 90 | 91 | destination_list = DestinationList( 92 | conf, [good_destination, bad_destination], cb, rl, 93 | persistent_launch_tor) 94 | functional_destinations = destination_list.functional_destinations 95 | assert [good_destination] == functional_destinations 96 | -------------------------------------------------------------------------------- /tests/integration/lib/test_relaylist.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | def test_relay_properties(rl): 4 | relay = [relay for relay in rl.relays if relay.nickname == 'test000a'][0] 5 | # The fingerprint and the master key can't be tested cause they are 6 | # created by chutney. 7 | assert 'Authority' in relay.flags 8 | assert not relay.exit_policy or not relay.exit_policy.is_exiting_allowed() 9 | assert relay.average_bandwidth == 1073741824 10 | assert relay.consensus_bandwidth == 0 11 | assert relay.address == '127.0.0.1' 12 | 13 | 14 | def test_relay_list_last_consensus_timestamp(rl): 15 | assert rl.last_consensus_timestamp == \ 16 | rl._relays[0].last_consensus_timestamp 17 | -------------------------------------------------------------------------------- /tests/integration/lib/test_relayprioritizer.py: -------------------------------------------------------------------------------- 1 | from sbws.lib.resultdump import ResultDump 2 | from sbws.lib.resultdump import ResultSuccess, ResultErrorCircuit 3 | from sbws.lib.relayprioritizer import RelayPrioritizer 4 | from unittest.mock import patch 5 | 6 | from sbws import settings 7 | 8 | 9 | def static_time(value): 10 | while True: 11 | yield value 12 | 13 | 14 | def _build_result_for_relay(conf, rl, result_type, relay_nick, 15 | timestamp): 16 | relay = [r for r in rl.relays if r.nickname == relay_nick] 17 | assert len(relay) == 1 18 | relay = relay[0] 19 | other = [r for r in rl.relays if r.nickname != relay_nick][0] 20 | circ = [relay.fingerprint, other.fingerprint] 21 | rtts = [0.5, 0.5, 0.5] 22 | dls = [ 23 | {'amount': 1024, 'duration': 1}, 24 | {'amount': 1024, 'duration': 1}, 25 | {'amount': 1024, 'duration': 1}, 26 | ] 27 | if result_type == ResultSuccess: 28 | return ResultSuccess(rtts, dls, relay, circ, 29 | conf['destinations.foo']['url'], 30 | 'test', t=timestamp) 31 | 32 | elif result_type == ResultErrorCircuit: 33 | return ResultErrorCircuit(relay, circ, 34 | conf['destinations.foo']['url'], 35 | 'test', msg='Test error circ message', 36 | t=timestamp) 37 | 38 | 39 | @patch('time.time') 40 | def test_relayprioritizer_general(time_mock, sbwshome_empty, args, 41 | conf, rl, 42 | persistent_launch_tor): 43 | now = 1000000 44 | time_mock.side_effect = static_time(now) 45 | rd = ResultDump(args, conf) 46 | try: 47 | rp = RelayPrioritizer(args, conf, rl, rd) 48 | results = [ 49 | _build_result_for_relay(conf, rl, ResultSuccess, 50 | 'test{:03d}m'.format(i), now - (i * 100)) 51 | # In chutney the relays are from 003 to 011 52 | for i in range(3, 12) 53 | ] 54 | for result in results: 55 | rd.store_result(result) 56 | best_list = list(rp.best_priority()) 57 | # With chutney, the relays not measured, with higher priority, will be 58 | # the 3 exits and authorities. 59 | # So take the list from the first measured relay, ie. from the 6th 60 | # position. 61 | # The measured relays will be in inverse order to their name. 62 | best_list_measured = best_list[6:] 63 | for i in range(3, 12): 64 | nick = 'test{:03d}m'.format(i) 65 | # -1 To start by the back, - 2 because their names start by 3, 66 | # not 1 67 | pos = (i - 2) * -1 68 | relay = best_list_measured[pos] 69 | assert relay.nickname == nick 70 | assert relay.relay_recent_priority_list_count == 1 71 | finally: 72 | settings.end_event.set() 73 | -------------------------------------------------------------------------------- /tests/integration/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Instead of exiting inmmediately when any of this commands fail, 3 | # the scanner, generate and coverage lines could continue and store there was 4 | # an error on that command. It's just simpler with `-e`. 5 | set -ex 6 | 7 | tests/integration/start_chutney.sh 8 | python3 scripts/tools/sbws-http-server.py --port 28888 &>/dev/null & 9 | sleep 1 10 | wget -O/dev/null http://127.0.0.1:28888/sbws.bin 11 | # Run actually the scanner 12 | sbws -c tests/integration/sbws_testnet.ini scanner 13 | sbws -c tests/integration/sbws_testnet.ini generate 14 | # Run integration tests 15 | coverage run -a --rcfile=.coveragerc --source=sbws -m pytest -s tests/integration -vv 16 | sbws -c tests/integration/sbws_testnet.ini cleanup 17 | tests/integration/stop_chutney.sh 18 | -------------------------------------------------------------------------------- /tests/integration/sbws_testnet.ini: -------------------------------------------------------------------------------- 1 | [general] 2 | # Days into the past that measurements are considered valid 3 | data_period = 1 4 | 5 | [paths] 6 | sbws_home = /tmp/.sbws 7 | 8 | [scanner] 9 | country = ZZ 10 | 11 | [destinations] 12 | local = on 13 | 14 | [destinations.local] 15 | ; url = https://localhost:28888/sbws.bin 16 | url = http://127.0.0.1:28888/sbws.bin 17 | verify = False 18 | country = ZZ 19 | 20 | [tor] 21 | external_control_port = 8015 22 | 23 | [logging] 24 | level = debug 25 | to_stdout_level = ${level} 26 | -------------------------------------------------------------------------------- /tests/integration/start_chutney.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -x 3 | 4 | CURRENT_DIR=`pwd` 5 | CHUTNEY_DIR=${1:-./chutney} 6 | 7 | # If chutney dir already exists, this will fail but it doesn't matter. 8 | git clone https://git.torproject.org/chutney.git $CHUTNEY_DIR 9 | 10 | cp tests/integration/chutney_data/bwscanner $CHUTNEY_DIR/networks 11 | cp tests/integration/chutney_data/*.tmpl $CHUTNEY_DIR/torrc_templates 12 | 13 | cd $CHUTNEY_DIR 14 | # In case it wasn't cloned recently, pull. 15 | # Since this is run only for the tests, it's ok if the tests fail with a newer 16 | # chutney version, so that we can detect it early. 17 | git pull 18 | 19 | # Stop chutney network if it is already running 20 | ./chutney stop networks/bwscanner 21 | ./chutney configure networks/bwscanner 22 | ./chutney start networks/bwscanner 23 | ./chutney status networks/bwscanner 24 | ./chutney wait_for_bootstrap networks/bwscanner 25 | 26 | # temporal workaround for https://gitlab.torproject.org/tpo/core/chutney/-/issues/40016 27 | sleep 60 28 | cd $CURRENT_DIR 29 | -------------------------------------------------------------------------------- /tests/integration/stop_chutney.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | 5 | CURRENT_DIR=`pwd` 6 | CHUTNEY_DIR=${1:-./chutney} 7 | cd $CHUTNEY_DIR 8 | # Stop chutney network if it is already running 9 | ./chutney stop networks/bwscanner 10 | cd $CURRENT_DIR 11 | -------------------------------------------------------------------------------- /tests/integration/test_files.py: -------------------------------------------------------------------------------- 1 | """ 2 | Integration tests for the files with data to be used by the bandwidth file. 3 | 4 | """ 5 | from sbws.lib.resultdump import load_recent_results_in_datadir 6 | from sbws.lib.v3bwfile import V3BWFile 7 | from sbws.util.state import State 8 | 9 | 10 | def test_results(conf): 11 | results = load_recent_results_in_datadir(5, conf["paths"]["datadir"]) 12 | for fp, values in results.items(): 13 | count = max( 14 | [ 15 | len(getattr(r, "relay_recent_measurement_attempt", [])) 16 | for r in values 17 | ] 18 | ) 19 | assert count == 1 20 | count = max( 21 | [len(getattr(r, "relay_in_recent_consensus", [])) for r in values] 22 | ) 23 | assert count == 1 24 | count = max( 25 | [len(getattr(r, "relay_recent_priority_list", [])) for r in values] 26 | ) 27 | assert count == 1 28 | 29 | 30 | def test_state(conf): 31 | state = State(conf["paths"]["state_fpath"]) 32 | assert 1 == state.count("recent_consensus") 33 | assert 1 == state.count("recent_priority_list") 34 | assert 15 == state.count("recent_priority_relay") 35 | # Because of 40023, it's 0. Change to 15 if we store it again at some point 36 | # assert 15 == state.count("recent_measurement_attempt") 37 | assert not state.count("recent_measurement_attempt") 38 | 39 | 40 | def test_v3bwfile(conf): 41 | bwfile = V3BWFile.from_v1_fpath( 42 | conf["paths"]["v3bw_fname"].format("latest") 43 | ) 44 | assert "1" == bwfile.header.recent_consensus_count 45 | assert "1" == bwfile.header.recent_priority_list_count 46 | assert "15" == bwfile.header.recent_priority_relay_count 47 | # Because of 40023, there is not this header 48 | # assert "15" == bwfile.header.recent_measurement_attempt_count 49 | for bwline in bwfile.bw_lines: 50 | assert 1 == bwline.relay_in_recent_consensus_count 51 | assert 1 == bwline.relay_recent_priority_list_count 52 | assert 1 == bwline.relay_recent_measurement_attempt_count 53 | -------------------------------------------------------------------------------- /tests/integration/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torproject/sbws/fd139b89de20b6d38941f137325e05069995793b/tests/integration/util/__init__.py -------------------------------------------------------------------------------- /tests/integration/util/test_requests.py: -------------------------------------------------------------------------------- 1 | """Integration tests for requests.""" 2 | import requests 3 | import uuid 4 | 5 | from sbws import settings 6 | from sbws.util import requests as requests_utils 7 | 8 | 9 | def test_make_session(conf, persistent_launch_tor, dests): 10 | uuid_str = str(uuid.uuid4()) 11 | settings.init_http_headers(conf.get('scanner', 'nickname'), uuid_str, 12 | str(persistent_launch_tor.get_version())) 13 | session = requests_utils.make_session( 14 | persistent_launch_tor, conf.getfloat('general', 'http_timeout')) 15 | assert session._timeout == conf.getfloat('general', 'http_timeout') 16 | 17 | # Because there is not an stream attached to a circuit, this will timeout. 18 | response = None 19 | try: 20 | response = session.get(dests.next().url, verify=False) 21 | except requests.exceptions.ConnectTimeout: 22 | pass 23 | assert response is None 24 | -------------------------------------------------------------------------------- /tests/integration/util/test_stem.py: -------------------------------------------------------------------------------- 1 | import sbws.util.stem as stem_utils 2 | 3 | 4 | def test_launch_and_okay(persistent_launch_tor): 5 | cont = persistent_launch_tor 6 | assert stem_utils.is_bootstrapped(cont) 7 | 8 | 9 | def test_set_torrc_runtime_option_succesful(persistent_launch_tor): 10 | controller = persistent_launch_tor 11 | runtime_options = controller.get_conf_map(['__LeaveStreamsUnattached']) 12 | assert runtime_options == {'__LeaveStreamsUnattached': ['1']} 13 | 14 | 15 | def test_set_torrc_runtime_invalidrequest_option_fail(persistent_launch_tor): 16 | controller = persistent_launch_tor 17 | try: 18 | controller.set_conf('ControlSocket', '/tmp/dummy') 19 | except stem_utils.InvalidRequest as e: 20 | assert "Unable to set option" in e.message 21 | 22 | 23 | def test_set_torrc_options_can_fail_option_fail(persistent_launch_tor): 24 | controller = persistent_launch_tor 25 | try: 26 | controller.set_conf('BadOption', '0') 27 | except stem_utils.InvalidArguments as e: 28 | assert "Unknown option" in e.message 29 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torproject/sbws/fd139b89de20b6d38941f137325e05069995793b/tests/unit/__init__.py -------------------------------------------------------------------------------- /tests/unit/core/test_generate.py: -------------------------------------------------------------------------------- 1 | """Unit tests for sbws.core.generate module.""" 2 | import argparse 3 | 4 | from sbws.globals import TORFLOW_ROUND_DIG, PROP276_ROUND_DIG 5 | from sbws.core.generate import gen_parser 6 | 7 | 8 | def test_gen_parser_arg_round_digs(): 9 | """ 10 | Test that both --torflow-round-digs and --round-digs arguments can be 11 | passed and round-digs is PROP276_ROUND_DIG by default. 12 | 13 | """ 14 | parent_parser = argparse.ArgumentParser(prog='sbws') 15 | subparsers = parent_parser.add_subparsers(help='generate help') 16 | parser_generate = gen_parser(subparsers) 17 | # Explicitely set empty arguments, otherwise pytest will use pytest 18 | # arguments 19 | args = parser_generate.parse_args([]) 20 | assert args.round_digs == PROP276_ROUND_DIG 21 | # torflow_round_digs is not in the Namespace 22 | assert getattr(args, 'torflow_round_digs', None) is None 23 | # but it can still be passed as an argument 24 | args = parser_generate.parse_args(['--torflow-round-digs', 25 | str(TORFLOW_ROUND_DIG)]) 26 | # though the variable is named round_digs 27 | assert args.round_digs == TORFLOW_ROUND_DIG 28 | # or use the short version 29 | args = parser_generate.parse_args(['-r', str(TORFLOW_ROUND_DIG)]) 30 | assert args.round_digs == TORFLOW_ROUND_DIG 31 | # or use round-digs explicitely 32 | args = parser_generate.parse_args(['--round-digs', 33 | str(PROP276_ROUND_DIG)]) 34 | assert args.round_digs == PROP276_ROUND_DIG 35 | -------------------------------------------------------------------------------- /tests/unit/core/test_scanner.py: -------------------------------------------------------------------------------- 1 | """Unit tests for scanner.py.""" 2 | import pytest 3 | 4 | from sbws.core.scanner import result_putter 5 | 6 | 7 | def test_result_putter(sbwshome_only_datadir, result_success, rd, end_event): 8 | if rd is None: 9 | pytest.skip("ResultDump is None") 10 | # Put one item in the queue 11 | callback = result_putter(rd) 12 | callback(result_success) 13 | assert rd.queue.qsize() == 1 14 | 15 | # Make queue maxsize 1, so that it'll be full after the first callback. 16 | # The second callback will wait 1 second, then the queue will be empty 17 | # again. 18 | rd.queue.maxsize = 1 19 | callback(result_success) 20 | # after putting 1 result, the queue will be full 21 | assert rd.queue.qsize() == 1 22 | assert rd.queue.full() 23 | # it's still possible to put another results, because the callback will 24 | # wait 1 second and the queue will be empty again. 25 | callback(result_success) 26 | assert rd.queue.qsize() == 1 27 | assert rd.queue.full() 28 | end_event.set() 29 | -------------------------------------------------------------------------------- /tests/unit/core/test_stats.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | 3 | import sbws.core.stats 4 | from tests.unit.globals import monotonic_time 5 | from unittest.mock import patch 6 | import logging 7 | 8 | 9 | def test_stats_initted(sbwshome_empty, args, conf, caplog): 10 | ''' 11 | An initialized but rather empty .sbws directory should fail about missing 12 | ~/.sbws/datadir 13 | ''' 14 | try: 15 | sbws.core.stats.main(args, conf) 16 | except SystemExit as e: 17 | assert e.code == 1 18 | else: 19 | assert None, 'Should have failed' 20 | assert '{}/datadir does not exist'.format( 21 | os.path.abspath(sbwshome_empty)) == caplog.records[-1].getMessage() 22 | 23 | 24 | def test_stats_stale_result(args, conf, caplog, 25 | sbwshome_success_result): 26 | ''' 27 | An initialized .sbws directory with no fresh results should say so and 28 | exit cleanly 29 | ''' 30 | caplog.set_level(logging.DEBUG) 31 | sbws.core.stats.main(args, conf) 32 | assert 'No fresh results' == caplog.records[-1].getMessage() 33 | 34 | 35 | @patch('time.time') 36 | def test_stats_fresh_result(time_mock, sbwshome_error_result, args, conf, 37 | capsys, caplog): 38 | ''' 39 | An initialized .sbws directory with a fresh error result should have some 40 | boring stats and exit cleanly 41 | ''' 42 | args.error_types = False 43 | start = 1529232278 44 | time_mock.side_effect = monotonic_time(start=start) 45 | sbws.core.stats.main(args, conf) 46 | captured = capsys.readouterr() 47 | lines = captured.out.strip().split('\n') 48 | assert '1 relays have recent results' in lines[0] 49 | # FIXME 50 | # needed_output_lines = [ 51 | # '1 relays have recent results', 52 | # 'Mean 0.00 successful measurements per relay', 53 | # '0 success results and 1 error results', 54 | # ] 55 | # for needed_line in needed_output_lines: 56 | # assert needed_line in lines 57 | # lines = [l.getMessage() for l in caplog.records] 58 | # needed_log_lines = [ 59 | # 'Keeping 1/1 read lines from {}/{}/{}.txt'.format( 60 | # sbwshome_error_result, 'datadir', '2018-06-17'), 61 | # 'Keeping 1/1 results after removing old ones', 62 | # ] 63 | # for needed_line in needed_log_lines: 64 | # assert needed_line in lines 65 | 66 | 67 | @patch('time.time') 68 | def test_stats_fresh_results(time_mock, sbwshome_success_result_two_relays, 69 | args, conf, capsys, caplog): 70 | ''' 71 | An initialized .sbws directory with a fresh error and fresh success should 72 | have some exciting stats and exit cleanly 73 | ''' 74 | caplog.set_level(logging.DEBUG) 75 | start = 1529232278 76 | time_mock.side_effect = monotonic_time(start=start) 77 | sbws.core.stats.main(args, conf) 78 | captured = capsys.readouterr() 79 | lines = captured.out.strip().split('\n') 80 | assert '1 relays have recent results' in lines[0] 81 | # FIXME 82 | # needed_output_lines = [ 83 | # '1 relays have recent results', 84 | # '1 success results and 1 error results', 85 | # 'Mean 1.00 successful measurements per relay', 86 | # '1/2 (50.00%) results were error-misc', 87 | # ] 88 | # for needed_line in needed_output_lines: 89 | # assert needed_line in lines 90 | # lines = [l.getMessage() for l in caplog.records] 91 | # needed_log_lines = [ 92 | # 'Keeping 2/2 read lines from {}/{}/{}.txt'.format( 93 | # sbwshome_success_result_two_relays, 'datadir', 94 | # datetime.utcfromtimestamp(time.time()).date()), 95 | # 'Keeping 2/2 results after removing old ones', 96 | # 'Found a _ResultType.Error for the first time', 97 | # 'Found a _ResultType.Success for the first time', 98 | # ] 99 | # for needed_line in needed_log_lines: 100 | # assert needed_line in lines 101 | -------------------------------------------------------------------------------- /tests/unit/globals.py: -------------------------------------------------------------------------------- 1 | def incrementing_time(start=2000, increment=1): 2 | while True: 3 | yield start 4 | start += increment 5 | 6 | 7 | def monotonic_time(start=2000): 8 | return incrementing_time(start, increment=0.000001) 9 | 10 | 11 | def static_time(value): 12 | while True: 13 | yield value 14 | -------------------------------------------------------------------------------- /tests/unit/lib/data/results.txt: -------------------------------------------------------------------------------- 1 | {"version": 4, "time": 1523974147, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "error-stream", "msg": "Something bad happened while measuring bandwidth", "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s"} 2 | {"version": 4, "time": 1523887747, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "success", "rtts": [0.4596822261810303, 0.44872617721557617, 0.4563450813293457, 0.44872212409973145, 0.4561030864715576, 0.4765200614929199, 0.4495084285736084, 0.45711588859558105, 0.45520496368408203, 0.4635589122772217], "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "downloads": [{"amount": 590009, "duration": 6.1014368534088135}, {"amount": 590009, "duration": 8.391342878341675}, {"amount": 321663, "duration": 7.064587831497192}, {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", "relay_average_bandwidth": 1000000000, "relay_burst_bandwidth": 123456, "relay_observed_bandwidth": 524288, "consensus_bandwidth": 600000, "consensus_bandwidth_is_unmeasured": false, "relay_in_recent_consensus": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_measurement_attempt": ["2018-04-07T14:09:07", "2018-04-08T14:09:07"], "relay_recent_priority_list": ["2018-04-07T14:09:07", "2018-04-08T14:09:07", "2018-04-09T14:09:07"]} 3 | {"version": 4, "time": 1523974147, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "error-stream", "msg": "Something bad happened while measuring bandwidth", "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", "relay_in_recent_consensus": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_measurement_attempt": ["2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_priority_list": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"]} 4 | -------------------------------------------------------------------------------- /tests/unit/lib/data/results_0_consensus_bw.txt: -------------------------------------------------------------------------------- 1 | {"version": 4, "time": 1523887747, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "success", "rtts": [0.4596822261810303, 0.44872617721557617, 0.4563450813293457, 0.44872212409973145, 0.4561030864715576, 0.4765200614929199, 0.4495084285736084, 0.45711588859558105, 0.45520496368408203, 0.4635589122772217], "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "downloads": [{"amount": 590009, "duration": 6.1014368534088135}, {"amount": 590009, "duration": 8.391342878341675}, {"amount": 321663, "duration": 7.064587831497192}, {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", "relay_burst_bandwidth": 1000000000, "relay_observed_bandwidth": 524288, "relay_average_bandwidth": 500000000, "consensus_bandwidth": 0, "consensus_bandwidth_is_unmeasured": false, "relay_in_recent_consensus": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_measurement_attempt": ["2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_priority_list": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"]} 2 | -------------------------------------------------------------------------------- /tests/unit/lib/data/results_no_consensus_bw.txt: -------------------------------------------------------------------------------- 1 | {"version": 4, "time": 1523887747, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "success", "rtts": [0.4596822261810303, 0.44872617721557617, 0.4563450813293457, 0.44872212409973145, 0.4561030864715576, 0.4765200614929199, 0.4495084285736084, 0.45711588859558105, 0.45520496368408203, 0.4635589122772217], "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "downloads": [{"amount": 590009, "duration": 6.1014368534088135}, {"amount": 590009, "duration": 8.391342878341675}, {"amount": 321663, "duration": 7.064587831497192}, {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", "relay_burst_bandwidth": 1000000000, "relay_observed_bandwidth": 524288, "relay_average_bandwidth": 500000000, "consensus_bandwidth": null, "consensus_bandwidth_is_unmeasured": false, "relay_in_recent_consensus": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_measurement_attempt": ["2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_priority_list": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"]} 2 | -------------------------------------------------------------------------------- /tests/unit/lib/data/results_no_desc_bw_avg.txt: -------------------------------------------------------------------------------- 1 | {"version": 4, "time": 1523887747, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "success", "rtts": [0.4596822261810303, 0.44872617721557617, 0.4563450813293457, 0.44872212409973145, 0.4561030864715576, 0.4765200614929199, 0.4495084285736084, 0.45711588859558105, 0.45520496368408203, 0.4635589122772217], "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "downloads": [{"amount": 590009, "duration": 6.1014368534088135}, {"amount": 590009, "duration": 8.391342878341675}, {"amount": 321663, "duration": 7.064587831497192}, {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", "relay_burst_bandwidth": 1000000000, "relay_observed_bandwidth": 524288, "relay_average_bandwidth": null, "consensus_bandwidth": 600000, "consensus_bandwidth_is_unmeasured": false, "relay_in_recent_consensus": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_measurement_attempt": ["2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_priority_list": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"]} 2 | -------------------------------------------------------------------------------- /tests/unit/lib/data/results_no_desc_bw_avg_obs.txt: -------------------------------------------------------------------------------- 1 | {"version": 4, "time": 1523887747, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "success", "rtts": [0.4596822261810303, 0.44872617721557617, 0.4563450813293457, 0.44872212409973145, 0.4561030864715576, 0.4765200614929199, 0.4495084285736084, 0.45711588859558105, 0.45520496368408203, 0.4635589122772217], "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "downloads": [{"amount": 590009, "duration": 6.1014368534088135}, {"amount": 590009, "duration": 8.391342878341675}, {"amount": 321663, "duration": 7.064587831497192}, {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", "relay_burst_bandwidth": 1000000000, "relay_observed_bandwidth": null, "relay_average_bandwidth": null, "consensus_bandwidth": 600000, "consensus_bandwidth_is_unmeasured": false, "relay_in_recent_consensus": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_measurement_attempt": ["2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_priority_list": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"]} 2 | -------------------------------------------------------------------------------- /tests/unit/lib/data/results_no_desc_bw_obs.txt: -------------------------------------------------------------------------------- 1 | {"version": 4, "time": 1523887747, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "success", "rtts": [0.4596822261810303, 0.44872617721557617, 0.4563450813293457, 0.44872212409973145, 0.4561030864715576, 0.4765200614929199, 0.4495084285736084, 0.45711588859558105, 0.45520496368408203, 0.4635589122772217], "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "downloads": [{"amount": 590009, "duration": 6.1014368534088135}, {"amount": 590009, "duration": 8.391342878341675}, {"amount": 321663, "duration": 7.064587831497192}, {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", "relay_burst_bandwidth": 1000000000, "relay_observed_bandwidth": null, "relay_average_bandwidth": 500000000, "consensus_bandwidth": 600000, "consensus_bandwidth_is_unmeasured": false, "relay_in_recent_consensus": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_measurement_attempt": ["2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_priority_list": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"]} 2 | -------------------------------------------------------------------------------- /tests/unit/lib/test_destination.py: -------------------------------------------------------------------------------- 1 | """Unit tests for sbws.lib.destination.""" 2 | from datetime import datetime, timedelta 3 | 4 | from sbws.globals import MAX_SECONDS_RETRY_DESTINATION 5 | from sbws.lib import destination 6 | 7 | 8 | def test_destination_is_functional(): 9 | eleven_mins_ago = datetime.utcnow() - timedelta(minutes=11) 10 | six_mins_ago = datetime.utcnow() - timedelta(minutes=6) 11 | four_mins_ago = datetime.utcnow() - timedelta(minutes=4) 12 | # Make last time tried a bit bigger than the half of the maximum, so that 13 | # it's bigger than the delta time to retry, and when delta time to retry 14 | # is muliplied by a factor (2) it reaches the maximum. 15 | long_ago = datetime.utcnow() - timedelta( 16 | (MAX_SECONDS_RETRY_DESTINATION / 2) + 2 17 | ) 18 | 19 | d = destination.Destination('unexistenturl', 0, False) 20 | assert d.is_functional() 21 | 22 | # Fail 3 consecutive times 23 | d.add_failure() 24 | d.add_failure() 25 | d.add_failure() 26 | assert d._are_last_attempts_failures() 27 | assert not d._is_last_try_old_enough() 28 | assert not d.is_functional() 29 | 30 | # Then doesn't fail and it's functional again 31 | d.add_success() 32 | assert not d._are_last_attempts_failures() 33 | assert d.is_functional() 34 | 35 | # Fail again 3 times 36 | d.add_failure() 37 | d.add_failure() 38 | # And last failure was 2h ago 39 | d.add_failure(four_mins_ago) 40 | assert d._are_last_attempts_failures() 41 | assert not d._is_last_try_old_enough() 42 | assert not d.is_functional() 43 | 44 | # But if the last failure was 4h ago, try to use it again 45 | # And last failure was 4h ago 46 | d.add_failure(six_mins_ago) 47 | assert d._is_last_try_old_enough() 48 | assert d.is_functional() 49 | 50 | # If last failure was 8h ago, try to use it again again 51 | d.add_failure(eleven_mins_ago) 52 | assert d._is_last_try_old_enough() 53 | assert d.is_functional() 54 | 55 | # Whenever it does not fail again, reset the time to try again 56 | # on 3 consecutive failures 57 | d.add_success() 58 | assert not d._are_last_attempts_failures() 59 | assert d.is_functional() 60 | # And the delta to try is resetted 61 | assert not d._is_last_try_old_enough() 62 | 63 | # When the delta time to retry a destination increase too much, 64 | # set it to a maximum, and try the destination again 65 | d.add_failure() 66 | d.add_failure() 67 | d.add_failure(long_ago) 68 | # Pretend the delta seconds was already set to a bit more than 69 | # half the maximum. 70 | d._delta_seconds_retry = (MAX_SECONDS_RETRY_DESTINATION / 2) + 1 71 | assert d._are_last_attempts_failures() 72 | assert d._is_last_try_old_enough() 73 | assert d.is_functional() 74 | assert d._delta_seconds_retry == MAX_SECONDS_RETRY_DESTINATION 75 | -------------------------------------------------------------------------------- /tests/unit/lib/test_heartbeat.py: -------------------------------------------------------------------------------- 1 | """Unit tests for heartbeat""" 2 | import logging 3 | import pytest 4 | 5 | from sbws.lib import heartbeat 6 | from sbws.util.state import State 7 | 8 | 9 | @pytest.mark.skip(reason="increment_recent_measurement_attempt() disabled") 10 | def test_total_measured_percent(conf, caplog): 11 | state = State(conf["paths"]["state_fname"]) 12 | state["recent_priority_list"] = [1, 2, 3] 13 | hbeat = heartbeat.Heartbeat(conf.getpath('paths', 'state_fname')) 14 | 15 | hbeat.register_consensus_fprs(['A', 'B', 'C']) 16 | 17 | hbeat.register_measured_fpr('A') 18 | hbeat.register_measured_fpr('B') 19 | 20 | caplog.set_level(logging.INFO) 21 | 22 | assert hbeat.previous_measurement_percent == 0 23 | 24 | hbeat.print_heartbeat_message() 25 | 26 | assert hbeat.previous_measurement_percent == 67 27 | assert 0 == caplog.records[0].getMessage().find("Run 3 main loops.") 28 | assert 0 == caplog.records[1].getMessage().find( 29 | "Measured in total 2 (67%)" 30 | ) 31 | assert 0 == caplog.records[2].getMessage().find( 32 | "1 relays still not measured" 33 | ) 34 | -------------------------------------------------------------------------------- /tests/unit/lib/test_relayprioritizer.py: -------------------------------------------------------------------------------- 1 | """relayprioritizer.py unit tests.""" 2 | from freezegun import freeze_time 3 | 4 | 5 | def test_increment_recent_priority_list(relay_prioritizer): 6 | """Test that incrementing the priority lists do not go on forever. 7 | 8 | And instead it only counts the number of priority lists in the last days. 9 | """ 10 | 11 | state = relay_prioritizer._state 12 | assert 0 == relay_prioritizer.recent_priority_list_count 13 | assert not state.get("recent_priority_list", None) 14 | 15 | # Pretend that a priority list is made. 16 | with freeze_time("2020-02-29 10:00:00"): 17 | relay_prioritizer.increment_recent_priority_list() 18 | assert 1 == relay_prioritizer.recent_priority_list_count 19 | assert 1 == len(state["recent_priority_list"]) 20 | 21 | # And a second priority list is made 4 days later. 22 | with freeze_time("2020-03-04 10:00:00"): 23 | relay_prioritizer.increment_recent_priority_list() 24 | assert 2 == relay_prioritizer.recent_priority_list_count 25 | assert 2 == len(state["recent_priority_list"]) 26 | 27 | # And a third priority list is made 5 days later. 28 | with freeze_time("2020-03-05 10:00:00"): 29 | relay_prioritizer.increment_recent_priority_list() 30 | assert 3 == relay_prioritizer.recent_priority_list_count 31 | assert 3 == len(state["recent_priority_list"]) 32 | 33 | # And a fourth priority list is made 6 days later. The first one is 34 | # now removed and not counted. 35 | with freeze_time("2020-03-06 10:00:00"): 36 | relay_prioritizer.increment_recent_priority_list() 37 | assert 3 == relay_prioritizer.recent_priority_list_count 38 | assert 3 == len(state["recent_priority_list"]) 39 | 40 | 41 | def test_increment_priority_relay(relay_prioritizer): 42 | """Test that incrementing the number of relays in the priority lists 43 | do not go on forever. 44 | 45 | And instead it only counts number of relays in priority lists in the last 46 | days. 47 | """ 48 | 49 | state = relay_prioritizer._state 50 | assert 0 == relay_prioritizer.recent_priority_relay_count 51 | assert not state.get("recent_priority_relay", None) 52 | 53 | # Pretend that a priority list is made. 54 | with freeze_time("2020-02-29 10:00:00"): 55 | relay_prioritizer.increment_recent_priority_relay(2) 56 | assert 2 == relay_prioritizer.recent_priority_relay_count 57 | assert 2 == state.count("recent_priority_relay") 58 | 59 | # And a second priority list is made 4 days later. 60 | with freeze_time("2020-03-04 10:00:00"): 61 | relay_prioritizer.increment_recent_priority_relay(2) 62 | assert 4 == relay_prioritizer.recent_priority_relay_count 63 | assert 4 == state.count("recent_priority_relay") 64 | 65 | # And a third priority list is made 5 days later. 66 | with freeze_time("2020-03-05 10:00:00"): 67 | relay_prioritizer.increment_recent_priority_relay(2) 68 | assert 6 == relay_prioritizer.recent_priority_relay_count 69 | assert 6 == state.count("recent_priority_relay") 70 | 71 | # And a fourth priority list is made 6 days later. The first one is 72 | # now removed and the relays are not counted. 73 | with freeze_time("2020-03-06 10:00:00"): 74 | relay_prioritizer.increment_recent_priority_relay(2) 75 | assert 6 == relay_prioritizer.recent_priority_relay_count 76 | assert 6 == state.count("recent_priority_relay") 77 | -------------------------------------------------------------------------------- /tests/unit/lib/test_resultdump.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Unit tests for resultdump.""" 3 | 4 | import datetime 5 | import logging 6 | 7 | from sbws.lib.relaylist import Relay 8 | from sbws.lib.resultdump import ( 9 | ResultError, 10 | ResultErrorStream, 11 | ResultSuccess, 12 | trim_results_ip_changed, 13 | load_result_file 14 | ) 15 | 16 | 17 | def test_trim_results_ip_changed_defaults(resultdict_ip_not_changed): 18 | results_dict = trim_results_ip_changed(resultdict_ip_not_changed) 19 | assert resultdict_ip_not_changed == results_dict 20 | 21 | 22 | def test_trim_results_ip_changed_on_changed_ipv4_changed( 23 | resultdict_ip_changed, resultdict_ip_changed_trimmed): 24 | results_dict = trim_results_ip_changed(resultdict_ip_changed, 25 | on_changed_ipv4=True) 26 | assert resultdict_ip_changed_trimmed == results_dict 27 | 28 | 29 | def test_trim_results_ip_changed_on_changed_ipv4_no_changed( 30 | resultdict_ip_not_changed): 31 | results_dict = trim_results_ip_changed(resultdict_ip_not_changed, 32 | on_changed_ipv4=True) 33 | assert resultdict_ip_not_changed == results_dict 34 | 35 | 36 | def test_trim_results_ip_changed_on_changed_ipv6(caplog, 37 | resultdict_ip_not_changed): 38 | results_dict = trim_results_ip_changed(resultdict_ip_not_changed, 39 | on_changed_ipv6=True) 40 | assert resultdict_ip_not_changed == results_dict 41 | # There might be other logs from other threads. 42 | with caplog.at_level(logging.WARNING): 43 | assert 'Reseting bandwidth results when IPv6 changes, ' \ 44 | 'is not yet implemented.\n' in caplog.text 45 | 46 | 47 | def test_resultdump( 48 | rd, args, conf_results, controller, router_status, server_descriptor 49 | ): 50 | from sbws import settings 51 | relay = Relay( 52 | router_status.fingerprint, 53 | controller, 54 | ns=router_status, 55 | desc=server_descriptor, 56 | ) 57 | relay.increment_relay_recent_priority_list() 58 | relay.increment_relay_recent_measurement_attempt() 59 | r = ResultSuccess( 60 | [], 2000, relay, ["A", "B"], "http://localhost/bw", "scanner_nick", 61 | ) 62 | # Storing the result with `rd.queue.put` will not store the result to disk 63 | # because the thread is not spawned with pytest. 64 | rd.store_result(r) 65 | results = rd.results_for_relay(relay) 66 | # It has stored the result 67 | assert 1 == len(results) 68 | # The result has the correct attribute 69 | assert 1 == len(results[0].relay_recent_priority_list) 70 | # Store a second result for the sme relay 71 | r = ResultError( 72 | relay, ["A", "B"], "http://localhost/bw", "scanner_nick", 73 | ) 74 | rd.store_result(r) 75 | assert 2 == len(results) 76 | assert 1 == len(results[1].relay_recent_priority_list) 77 | settings.set_end_event() 78 | 79 | 80 | def test_load(datadir): 81 | results = load_result_file(str(datadir.join("results.txt"))) 82 | results = [v for values in results.values() for v in values] 83 | r1 = results[1] 84 | assert isinstance(r1, ResultSuccess) 85 | assert isinstance( 86 | r1.relay_recent_measurement_attempt[0], datetime.datetime 87 | ) 88 | assert 2 == len(r1.relay_recent_measurement_attempt) 89 | assert 3 == len(r1.relay_recent_priority_list) 90 | assert 3 == len(r1.relay_in_recent_consensus) 91 | r2 = results[2] 92 | assert isinstance(r2, ResultErrorStream) 93 | assert isinstance( 94 | r2.relay_recent_measurement_attempt[0], datetime.datetime 95 | ) 96 | assert 2 == len(r2.relay_recent_measurement_attempt) 97 | assert 3 == len(r2.relay_recent_priority_list) 98 | assert 3 == len(r2.relay_in_recent_consensus) 99 | -------------------------------------------------------------------------------- /tests/unit/lib/test_scaling.py: -------------------------------------------------------------------------------- 1 | """Unit tests for scaling.py.""" 2 | import os 3 | from statistics import mean 4 | 5 | from sbws.lib import scaling 6 | from sbws.lib.resultdump import load_result_file, ResultSuccess 7 | 8 | 9 | def test_bw_filt(): 10 | bw_measurements = [ 11 | 96700.00922329757, 70311.63051659254, 45531.743347556374, 12 | 38913.97025485627, 55656.332364676025 13 | ] 14 | fb = scaling.bw_filt(bw_measurements) 15 | # This is greater than the mean, that is 61422.73714139576 16 | assert fb == 83506 17 | 18 | # When there are no measurements what can not be the case for successful 19 | # results. 20 | bw_measurements = [] 21 | assert 0 == scaling.bw_filt(bw_measurements) 22 | 23 | bw_measurements = [1, 0] 24 | # Because rounded to int 25 | assert 0 == round(mean(bw_measurements)) 26 | # So the filtered bw will be also 0 27 | assert 0 == scaling.bw_filt(bw_measurements) 28 | 29 | bw_measurements = [1, 2, 3] 30 | # Because rounded to int 31 | assert 2 == round(mean(bw_measurements)) 32 | assert 2 == scaling.bw_filt(bw_measurements) 33 | 34 | bw_measurements = [10, 0] 35 | assert 5 == round(mean(bw_measurements)) 36 | # Because the value 10 is bigger than the mean 37 | assert 10 == scaling.bw_filt(bw_measurements) 38 | 39 | bw_measurements = [0, 10, 20] 40 | assert 10 == round(mean(bw_measurements)) 41 | # Because 10 and 20 are bigger or equal than the mean 42 | assert 15 == scaling.bw_filt(bw_measurements) 43 | 44 | 45 | def test_bw_filt_from_results(root_data_path): 46 | results_file = os.path.join( 47 | root_data_path, ".sbws", "datadir", "2019-03-25.txt" 48 | ) 49 | results = load_result_file(results_file) 50 | bw_filts = {} 51 | for fp, values in results.items(): 52 | success_results = [r for r in values if isinstance(r, ResultSuccess)] 53 | if success_results: 54 | bw_measurements = scaling.bw_measurements_from_results( 55 | success_results 56 | ) 57 | mu = round(mean(bw_measurements)) 58 | muf = scaling.bw_filt(bw_measurements) 59 | bw_filts[fp] = (mu, muf) 60 | for fp, values in bw_filts.items(): 61 | assert bw_filts[fp][0] <= bw_filts[fp][1] 62 | assert 5526756 == bw_filts['117A456C911114076BEB4E757AC48B16CC0CCC5F'][0] 63 | assert 5643086 == bw_filts['117A456C911114076BEB4E757AC48B16CC0CCC5F'][1] 64 | assert 5664965 == bw_filts['693F73187624BE760AAD2A12C5ED89DB1DE044F5'][0] 65 | assert 5774274 == bw_filts['693F73187624BE760AAD2A12C5ED89DB1DE044F5'][1] 66 | assert 5508279 == bw_filts['270A861ABED22EC2B625198BCCD7B2B9DBFFC93C'][0] 67 | assert 5583737 == bw_filts['270A861ABED22EC2B625198BCCD7B2B9DBFFC93C'][1] 68 | assert 5379911 == bw_filts['E894C65997F8EC96558B554176EEEA39C6A43EF6'][0] 69 | assert 5485088 == bw_filts['E894C65997F8EC96558B554176EEEA39C6A43EF6'][1] 70 | -------------------------------------------------------------------------------- /tests/unit/test_bwfile_health.py: -------------------------------------------------------------------------------- 1 | """Test that the KeyValues in a bandwidth file make sense.""" 2 | import os.path 3 | 4 | from sbws.lib.bwfile_health import BwFile 5 | 6 | 7 | def test_bwfile_health(root_data_path): 8 | bwfile = BwFile.load(os.path.join( 9 | root_data_path, "2020-03-22-08-35-00-bandwidth" 10 | )) 11 | assert bwfile.header.is_correct 12 | assert bwfile.are_bwlines_correct 13 | assert bwfile.is_correct 14 | 15 | 16 | def test_bwlines_health(capsys, root_data_path): 17 | bwfile = BwFile.load(os.path.join( 18 | root_data_path, "2020-03-22-08-35-00-bandwidth" 19 | )) 20 | out = ( 21 | "\nrelay_recent_measurement_attempt_count <= relay_recent_priority_list_count,\n" # noqa 22 | "True\n" 23 | "relay_recent_priority_list_count <= relay_recent_consensus_count,\n" 24 | "True\n\n" 25 | ) 26 | for bwline in bwfile.bwlines: 27 | bwline.report 28 | assert out == capsys.readouterr().out 29 | -------------------------------------------------------------------------------- /tests/unit/util/data/user_sbws.ini: -------------------------------------------------------------------------------- 1 | [paths] 2 | 3 | sbws_home = /tmp/.sbws -------------------------------------------------------------------------------- /tests/unit/util/test_json.py: -------------------------------------------------------------------------------- 1 | """json.py unit tests.""" 2 | import json 3 | 4 | from sbws.util.json import CustomDecoder, CustomEncoder 5 | 6 | STATE = """{ 7 | "min_perc_reached": null, 8 | "recent_consensus_count": [ 9 | "2020-03-04T10:00:00", 10 | "2020-03-05T10:00:00", 11 | "2020-03-06T10:00:00" 12 | ], 13 | "recent_measurement_attempt": [ 14 | [ 15 | "2020-03-04T10:00:00", 16 | 2 17 | ], 18 | [ 19 | "2020-03-05T10:00:00", 20 | 2 21 | ], 22 | [ 23 | "2020-03-06T10:00:00", 24 | 2 25 | ] 26 | ], 27 | "recent_priority_list": [ 28 | "2020-03-04T10:00:00", 29 | "2020-03-05T10:00:00", 30 | "2020-03-06T10:00:00" 31 | ], 32 | "recent_priority_relay": [ 33 | [ 34 | "2020-03-04T10:00:00", 35 | 2 36 | ], 37 | [ 38 | "2020-03-05T10:00:00", 39 | 2 40 | ], 41 | [ 42 | "2020-03-06T10:00:00", 43 | 2 44 | ] 45 | ], 46 | "scanner_started": "2020-03-14T16:15:22", 47 | "uuid": "x" 48 | }""" 49 | 50 | 51 | def test_decode_encode_roundtrip(): 52 | d = json.loads(STATE, cls=CustomDecoder) 53 | s = json.dumps(d, cls=CustomEncoder, indent=4, sort_keys=True) 54 | assert s == STATE 55 | -------------------------------------------------------------------------------- /tests/unit/util/test_state.py: -------------------------------------------------------------------------------- 1 | from sbws.util.state import State 2 | import os 3 | # from tempfile import NamedTemporaryFile as NTF 4 | 5 | 6 | def test_state_set_allowed_key_types(tmpdir): 7 | state = State(os.path.join(str(tmpdir), 'statefoo')) 8 | attempt_keys = ('k') 9 | for key in attempt_keys: 10 | state[key] = 4 11 | assert state[key] == 4 12 | 13 | 14 | def test_state_set_allowed_value_types(tmpdir): 15 | state = State(os.path.join(str(tmpdir), 'statefoo')) 16 | attempt_vals = (15983, None, True, -1.2, 'loooooool') 17 | for val in attempt_vals: 18 | state['foo'] = val 19 | assert state['foo'] == val 20 | 21 | 22 | def test_state_del(tmpdir): 23 | state = State(os.path.join(str(tmpdir), 'statefoo')) 24 | d = {'a': 1, 'b': 2, 'c': 3, 'd': 4} 25 | for key in d: 26 | state[key] = d[key] 27 | assert len(state) == len(d) 28 | 29 | del d['a'] 30 | del state['a'] 31 | assert len(state) == len(d) 32 | for key in d: 33 | assert d[key] == state[key] 34 | 35 | d['e'] = 5 36 | state['e'] = 5 37 | d['e'] = 5.5 38 | state['e'] = 5.5 39 | assert len(state) == len(d) 40 | 41 | 42 | def test_state_get_len(tmpdir): 43 | state = State(os.path.join(str(tmpdir), 'statefoo')) 44 | d = {'a': 1, 'b': 2, 'c': 3, 'd': 4} 45 | for key in d: 46 | state[key] = d[key] 47 | assert len(state) == len(d) 48 | 49 | del d['a'] 50 | del state['a'] 51 | assert len(state) == len(d) 52 | 53 | d['e'] = 5 54 | state['e'] = 5 55 | d['e'] = 5.5 56 | state['e'] = 5.5 57 | assert len(state) == len(d) 58 | 59 | 60 | def test_state_contains(tmpdir): 61 | state = State(os.path.join(str(tmpdir), 'statefoo')) 62 | d = {'a': 1, 'b': 2, 'c': 3, 'd': 4} 63 | for key in d: 64 | state[key] = d[key] 65 | assert 'a' in state 66 | assert 'e' not in state 67 | 68 | 69 | def test_state_iter(tmpdir): 70 | state = State(os.path.join(str(tmpdir), 'statefoo')) 71 | for key in state: 72 | pass 73 | d = {'a': 1, 'b': 2, 'c': 3, 'd': 4} 74 | for key in d: 75 | state[key] = d[key] 76 | assert set([key for key in state]) == set(d) 77 | 78 | 79 | def test_two_instances(tmpdir): 80 | """Test that 2 different intances don't overwrite each other""" 81 | s1 = State(os.path.join(str(tmpdir), 'state.dat')) 82 | s2 = State(os.path.join(str(tmpdir), 'state.dat')) 83 | s1["x"] = "foo" 84 | s2["y"] = "bar" 85 | assert s2["x"] == "foo" 86 | 87 | 88 | def test_datetime_values(tmpdir): 89 | import datetime 90 | state = State(os.path.join(str(tmpdir), 'state.dat')) 91 | now = datetime.datetime.utcnow().replace(microsecond=0) 92 | state["datetimes"] = now 93 | assert now == state["datetimes"] 94 | -------------------------------------------------------------------------------- /tests/unit/util/test_stem.py: -------------------------------------------------------------------------------- 1 | """Unit tests for stem.py""" 2 | 3 | from sbws.util.stem import parse_user_torrc_config 4 | 5 | 6 | def test_parse_user_torrc_config_new_keyvalue_options_success(): 7 | config_torrc_extra_lines = """ 8 | Log debug file /tmp/tor-debug.log 9 | NumCPUs 1 10 | """ 11 | torrc_dict = parse_user_torrc_config({}, config_torrc_extra_lines) 12 | assert torrc_dict == \ 13 | {'Log': 'debug file /tmp/tor-debug.log', 'NumCPUs': '1'} 14 | 15 | 16 | def test_parse_user_torrc_config_existing_keyvalue_options_fail(caplog): 17 | torrc_dict = {'SocksPort': 'auto'} 18 | config_torrc_extra_lines = """ 19 | SocksPort 9050 20 | """ 21 | torrc_dict_new = parse_user_torrc_config( 22 | torrc_dict, config_torrc_extra_lines) 23 | # the new dictionary contains the existing key option and a list with both 24 | # the existing value and the new value 25 | assert torrc_dict_new != torrc_dict 26 | assert torrc_dict_new == {'SocksPort': ['auto', '9050']} 27 | 28 | 29 | def test_parse_user_torrc_config_new_key_option_success(): 30 | config_torrc_extra_lines = """ 31 | LongLivedPorts 32 | """ 33 | torrc_dict = parse_user_torrc_config({}, config_torrc_extra_lines) 34 | assert torrc_dict == {'LongLivedPorts': None} 35 | -------------------------------------------------------------------------------- /tests/unit/util/test_timestamp.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Test timestamp conversion util functions""" 3 | from datetime import datetime, timezone, timedelta 4 | from sbws.util.timestamp import (dt_obj_to_isodt_str, unixts_to_dt_obj, 5 | unixts_to_isodt_str, unixts_to_str, is_old) 6 | 7 | 8 | isodt_str = '2018-05-23T12:55:04' 9 | dt_obj = datetime.strptime(isodt_str, '%Y-%m-%dT%H:%M:%S') 10 | unixts = int(dt_obj.replace(tzinfo=timezone.utc).timestamp()) 11 | 12 | 13 | def test_dt_obj_to_isodt_str(): 14 | assert isodt_str == dt_obj_to_isodt_str(dt_obj) 15 | 16 | 17 | def test_unixts_to_dt_obj(): 18 | assert dt_obj == unixts_to_dt_obj(unixts) 19 | 20 | 21 | def test_unixts_to_isodt_str(): 22 | assert isodt_str == unixts_to_isodt_str(unixts) 23 | 24 | 25 | def test_unixts_to_str(): 26 | assert str(unixts) == unixts_to_str(unixts) 27 | 28 | 29 | def test_is_old(): 30 | # Since this timestamp is generated a few microseconds before checking 31 | # the oldest timestamp, it will be old. 32 | old_timestamp = datetime.utcnow() - timedelta(days=5) 33 | assert is_old(old_timestamp) 34 | # A recent timestamp should be at least 1 second newer that the oldest 35 | recent_timestamp = datetime.utcnow() - timedelta(days=5) \ 36 | + timedelta(seconds=1) 37 | assert not is_old(recent_timestamp) 38 | -------------------------------------------------------------------------------- /tests/unit/util/test_timestamps.py: -------------------------------------------------------------------------------- 1 | """timestamps.py unit tests.""" 2 | 3 | from datetime import datetime, timedelta 4 | 5 | from sbws.util.state import State 6 | from sbws.util.timestamps import ( 7 | DateTimeSeq, 8 | DateTimeIntSeq, 9 | ) 10 | 11 | 12 | def test_update_datetime_seq(conf): 13 | now = datetime.utcnow().replace(microsecond=0) 14 | state = State(conf["paths"]["state_fpath"]) 15 | # Create a list of 6 datetimes that started 6 days in the past. 16 | dts = [now - timedelta(days=x) for x in range(6, 0, -1)] 17 | dt_seq = DateTimeSeq( 18 | dts, state=state, state_key="recent_measurement_attempt" 19 | ) 20 | new_dts = dt_seq.update() 21 | # The updated list will not contain the 2 first (left) datetimes and it 22 | # will have one last timestamp (right). 23 | assert new_dts[:-1] == dts[2:] 24 | assert 5 == state.count("recent_measurement_attempt") 25 | assert 5 == len(dt_seq) 26 | 27 | 28 | def test_update_datetime_int_seq(conf): 29 | now = datetime.utcnow().replace(microsecond=0) 30 | state = State(conf["paths"]["state_fpath"]) 31 | # Create a list of 6 datetimes that started 6 days in the past. 32 | dts = [[now - timedelta(days=x), 2] for x in range(6, 0, -1)] 33 | dt_seq = DateTimeIntSeq( 34 | dts, state=state, state_key="recent_measurement_attempt" 35 | ) 36 | new_dts = dt_seq.update() 37 | # The updated list will not contain the 2 first (left) tuples and it 38 | # will have one last tuple (right). 39 | # The last tuple have 0 as the integer, instead of 2, so the count will be 40 | # 2 * 4 = 8 41 | assert new_dts[:-1] == dts[2:] 42 | assert 8 == state.count("recent_measurement_attempt") 43 | # And `len` should return the same. 44 | assert 8 == len(dt_seq) 45 | -------------------------------------------------------------------------------- /tests/unit/util/test_userquery.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import patch 2 | from sbws.util.userquery import query_yes_no 3 | 4 | 5 | @patch('builtins.input') 6 | def test_userquery_missing_default_invalid_response(input_mock): 7 | input_mock.side_effect = [''] * 100 + ['k'] * 100 + ['yess'] * 100 +\ 8 | ['no o'] * 100 9 | try: 10 | query_yes_no('a?', default=None) 11 | except StopIteration: 12 | pass 13 | else: 14 | assert None, 'Should have looped forever (and StopItration been '\ 15 | 'thrown when we stopped feeding it empty responses)' 16 | assert input_mock.call_count == 401 17 | 18 | 19 | @patch('builtins.input') 20 | def test_userquery_missing_default_yes_response(input_mock): 21 | input_mock.side_effect = [''] * 100 + ['y'] 22 | assert query_yes_no('a?', default=None) 23 | assert input_mock.call_count == 101 24 | input_mock.reset_mock() 25 | 26 | input_mock.side_effect = [''] * 100 + ['Y'] 27 | assert query_yes_no('a?', default=None) 28 | assert input_mock.call_count == 101 29 | input_mock.reset_mock() 30 | 31 | input_mock.side_effect = [''] * 100 + ['Yes'] 32 | assert query_yes_no('a?', default=None) 33 | assert input_mock.call_count == 101 34 | input_mock.reset_mock() 35 | 36 | input_mock.side_effect = ['k'] * 100 + ['Yes'] 37 | assert query_yes_no('a?', default=None) 38 | assert input_mock.call_count == 101 39 | input_mock.reset_mock() 40 | 41 | input_mock.side_effect = ['k'] * 100 + ['Yes', 'No'] 42 | assert query_yes_no('a?', default=None) 43 | assert input_mock.call_count == 101 44 | input_mock.reset_mock() 45 | 46 | 47 | @patch('builtins.input') 48 | def test_userquery_missing_default_no_response(input_mock): 49 | input_mock.side_effect = [''] * 100 + ['n'] 50 | assert not query_yes_no('a?', default=None) 51 | assert input_mock.call_count == 101 52 | input_mock.reset_mock() 53 | 54 | input_mock.side_effect = [''] * 100 + ['N'] 55 | assert not query_yes_no('a?', default=None) 56 | assert input_mock.call_count == 101 57 | input_mock.reset_mock() 58 | 59 | input_mock.side_effect = [''] * 100 + ['No'] 60 | assert not query_yes_no('a?', default=None) 61 | assert input_mock.call_count == 101 62 | input_mock.reset_mock() 63 | 64 | input_mock.side_effect = ['k'] * 100 + ['No'] 65 | assert not query_yes_no('a?', default=None) 66 | assert input_mock.call_count == 101 67 | input_mock.reset_mock() 68 | 69 | input_mock.side_effect = ['k'] * 100 + ['No', 'Yes'] 70 | assert not query_yes_no('a?', default=None) 71 | assert input_mock.call_count == 101 72 | input_mock.reset_mock() 73 | 74 | 75 | @patch('builtins.input') 76 | def test_userquery_yes_default_invalid_response(input_mock): 77 | input_mock.side_effect = [''] * 100 78 | assert query_yes_no('a?', default='yes') 79 | assert input_mock.call_count == 1 80 | 81 | 82 | @patch('builtins.input') 83 | def test_userquery_no_default_invalid_response(input_mock): 84 | input_mock.side_effect = [''] * 100 85 | assert not query_yes_no('a?', default='no') 86 | assert input_mock.call_count == 1 87 | 88 | 89 | @patch('builtins.input') 90 | def test_userquery_bad_default_invalid_response(input_mock): 91 | input_mock.side_effect = [''] * 100 92 | try: 93 | query_yes_no('a?', default='nooo') 94 | except ValueError: 95 | pass 96 | else: 97 | assert None, 'Should not have allowed us to specify a bad default '\ 98 | 'value' 99 | assert input_mock.call_count == 0 100 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | skip_missing_interpreters = True 3 | envlist = py{36, 37, 38, 39, 310}, inst, setup, integration, lint, stats, doc 4 | 5 | [travis] 6 | python = 7 | 3.6: py36, inst, setup, unit, integration, lint, doc 8 | 3.7: py37, inst, setup, unit, integration, lint, doc 9 | 3.8: py38, inst, setup, unit, integration, lint, doc 10 | 3.9: py39, inst, setup, unit, integration, lint, doc 11 | nightly: pynightly, inst, setup, unit, integration, lint, doc 12 | 13 | ; [testenv] 14 | # install_command can be removed when --process-dependency-links is not 15 | # needed anymore, and this section commented 16 | # install_command = pip install {opts} {packages} 17 | 18 | # test that it can be installed with custom commands and clean env 19 | [testenv:inst] 20 | skip_install = True 21 | commands = 22 | # this will fail until --process-dependency-links is not needed 23 | # it needs to be commented since error code will be still 1 24 | - pip install . 25 | ignore_errors = True 26 | recreate = True 27 | 28 | [testenv:setup] 29 | skip_install = True 30 | # this will fail until --process-dependency-links is not needed 31 | # it needs to be commented since error code will be still 1 32 | commands = python setup.py install 33 | recreate = True 34 | 35 | [testenv] 36 | deps = .[test] 37 | commands = 38 | coverage run -a --rcfile={toxinidir}/.coveragerc --source=sbws -m pytest \ 39 | -s {toxinidir}/tests/unit -vv 40 | 41 | [testenv:integration] 42 | ignore_errors = True 43 | deps = .[test] 44 | whitelist_externals = 45 | bash 46 | commands = 47 | # For some reason .[test] is not copying config.* files 48 | pip install . 49 | bash -c tests/integration/run.sh {envtmpdir}/chutney 50 | 51 | [testenv:lint] 52 | skip_install = True 53 | deps = .[dev] 54 | commands = flake8 sbws scripts tests 55 | 56 | [testenv:clean] 57 | skip_install = True 58 | changedir={toxinidir} 59 | deps = coverage 60 | command = coverage erase 61 | 62 | [testenv:stats] 63 | skip_install = True 64 | changedir={toxinidir} 65 | deps = .[test] 66 | commands= 67 | # nothing to combine while not using several python versions 68 | # coverage combine 69 | coverage report 70 | coverage html 71 | 72 | [testenv:doc] 73 | deps = .[doc] 74 | whitelist_externals = make 75 | changedir = docs 76 | commands = 77 | make html 78 | # this requires build the pdf images 79 | # make latexpdf 80 | make man 81 | 82 | # this requires Internet, it should not be in envlist 83 | [testenv:doclinks] 84 | deps = .[doc] 85 | whitelist_externals = make 86 | changedir = docs 87 | commands = 88 | make linkcheck 89 | --------------------------------------------------------------------------------