├── .github └── FUNDING.yml ├── .gitignore ├── .travis.yml ├── AUTHORS ├── LICENSE ├── MANIFEST.in ├── README.md ├── appveyor.yml ├── pytest.ini ├── requirements-flake8.txt ├── requirements-test.txt ├── requirements.txt ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── integration │ ├── __init__.py │ └── test_integration.py └── unittest │ ├── __init__.py │ ├── data │ ├── dir_certs │ │ ├── cached-certs │ │ └── dir_cert_real │ ├── network_status │ │ ├── consensus │ │ ├── consensus.json │ │ ├── consensus_extra │ │ ├── consensus_extra.json │ │ ├── consensus_real │ │ └── consensus_real.json │ └── network_status_diff │ │ └── network-status-diff │ ├── test_consensus.py │ ├── test_crypto.py │ ├── test_documents.py │ ├── test_hiddenservice.py │ └── test_parsers.py ├── torpy ├── __init__.py ├── __main__.py ├── cache_storage.py ├── cell_socket.py ├── cells.py ├── circuit.py ├── cli │ ├── __init__.py │ ├── console.py │ └── socks.py ├── client.py ├── consesus.py ├── crypto.py ├── crypto_common.py ├── crypto_state.py ├── dirs.py ├── documents │ ├── __init__.py │ ├── basics.py │ ├── dir_key_certificate.py │ ├── factory.py │ ├── items.py │ ├── network_status.py │ └── network_status_diff.py ├── guard.py ├── hiddenservice.py ├── http │ ├── __init__.py │ ├── adapter.py │ ├── base.py │ ├── client.py │ ├── requests.py │ └── urlopener.py ├── keyagreement.py ├── parsers.py ├── stream.py └── utils.py └── tox.ini /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | custom: ['https://btc.com/16mF9TYaJKkb9eGbZ5jGuJbodTF3mYvcRF', 'https://coinrequest.io/request/anscJbCsqucdJ1m'] 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv*/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # Idea project 107 | .idea/ 108 | bak/ 109 | .isort.cfg 110 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | os: linux 2 | dist: bionic 3 | language: python 4 | python: 5 | - "3.6" 6 | - "3.7" 7 | - "3.8" 8 | - "3.9" 9 | install: 10 | - pip install tox-travis coveralls 11 | script: 12 | - tox -v 13 | after_success: 14 | - cd tests && coveralls 15 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | James Brown -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.md 2 | include LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Torpy ![Python Versions] [![Build Status](https://travis-ci.com/torpyorg/torpy.svg?branch=master)](https://travis-ci.com/torpyorg/torpy) [![Build status](https://ci.appveyor.com/api/projects/status/14l6t8nq4tvno1pg?svg=true)](https://ci.appveyor.com/project/jbrown299/torpy) [![Coverage Status](https://coveralls.io/repos/github/torpyorg/torpy/badge.svg?branch=master)](https://coveralls.io/github/torpyorg/torpy?branch=master) 2 | ===== 3 | 4 | A pure python Tor client implementation of the Tor protocol. 5 | Torpy can be used to communicate with clearnet hosts or hidden services through the [Tor Network](https://torproject.org/about/overview.html). 6 | 7 | **Features** 8 | - No Stem or official Tor client required 9 | - Support v2 hidden services ([v2 specification](https://gitweb.torproject.org/torspec.git/tree/rend-spec-v2.txt)) 10 | - Support *Basic* and *Stealth* authorization protocol 11 | - Provide simple [TorHttpAdapter](https://github.com/torpyorg/torpy/blob/master/torpy/http/adapter.py) for [requests](https://requests.readthedocs.io/) library 12 | - Provide simple urllib [tor_opener](https://github.com/torpyorg/torpy/blob/master/torpy/http/urlopener.py) for making requests without any dependencies 13 | - Provide simple Socks5 proxy 14 | 15 | **Donation** 16 | 17 | If you find this project interesting, you can send some [Bitcoins](https://bitcoin.org/) to address: `16mF9TYaJKkb9eGbZ5jGuJbodTF3mYvcRF` 18 | 19 | **Note** 20 | 21 | This product is produced independently from the Tor® anonymity software and carries no guarantee from [The Tor Project](https://www.torproject.org/) about quality, suitability or anything else. 22 | 23 | Console examples 24 | ----------- 25 | There are several console utilities to test the client. 26 | 27 | A simple HTTP/HTTPS request: 28 | ```bash 29 | $ torpy_cli --url https://ifconfig.me --header "User-Agent" "curl/7.37.0" 30 | Loading cached NetworkStatusDocument from TorCacheDirStorage: .local/share/torpy/network_status 31 | Loading cached DirKeyCertificateList from TorCacheDirStorage: .local/share/torpy/dir_key_certificates 32 | Connecting to guard node 141.98.136.79:443 (Poseidon; Tor 0.4.3.6)... (TorClient) 33 | Sending: GET https://ifconfig.me 34 | Creating new circuit #80000001 with 141.98.136.79:443 (Poseidon; Tor 0.4.3.6) router... 35 | ... 36 | Building 3 hops circuit... 37 | Extending the circuit #80000001 with 109.70.100.23:443 (kren; Tor 0.4.4.5)... 38 | ... 39 | Extending the circuit #80000001 with 199.249.230.175:443 (Quintex86; Tor 0.4.4.5)... 40 | ... 41 | Stream #4: creating attached to #80000001 circuit... 42 | Stream #4: connecting to ('ifconfig.me', 443) 43 | Stream #4: connected (remote ip '216.239.36.21') 44 | Stream #4: closing (state = Connected)... 45 | Stream #4: remote disconnected (reason = DONE) 46 | Response status: 200 47 | Stream #4: closing (state = Closed)... 48 | Stream #4: closed already 49 | Closing guard connections (TorClient)... 50 | Destroy circuit #80000001 51 | Closing guard connections (Router descriptor downloader)... 52 | Destroy circuit #80000002 53 | > 199.249.230.175 54 | ``` 55 | 56 | Create Socks5 proxy to relay requests via the Tor Network: 57 | ``` 58 | $ torpy_socks -p 1050 --hops 3 59 | Loading cached NetworkStatusDocument from TorCacheDirStorage: .local/share/torpy/network_status 60 | Connecting to guard node 89.142.75.60:9001 (spongebobness; Tor 0.3.5.8)... 61 | Creating new circuit #80000001 with 89.142.75.60:9001 (spongebobness; Tor 0.3.5.8) router... 62 | Building 3 hops circuit... 63 | Extending the circuit #80000001 with 185.248.143.42:9001 (torciusv; Tor 0.3.5.8)... 64 | Extending the circuit #80000001 with 158.174.122.199:9005 (che1; Tor 0.4.1.6)... 65 | Start socks proxy at 127.0.0.1:1050 66 | ... 67 | ``` 68 | 69 | Torpy module also has a command-line interface: 70 | 71 | ```bash 72 | $ python3.7 -m torpy --url https://facebookcorewwwi.onion --to-file index.html 73 | Loading cached NetworkStatusDocument from TorCacheDirStorage: .local/share/torpy/network_status 74 | Connecting to guard node 185.2.31.8:443 (cx10TorServer; Tor 0.4.0.5)... 75 | Sending: GET https://facebookcorewwwi.onion 76 | Creating new circuit #80000001 with 185.2.31.8:443 (cx10TorServer; Tor 0.4.0.5) router... 77 | Building 3 hops circuit... 78 | Extending the circuit #80000001 with 144.172.71.110:8447 (TonyBamanaboni; Tor 0.4.1.5)... 79 | Extending the circuit #80000001 with 179.43.134.154:9001 (father; Tor 0.4.0.5)... 80 | Creating stream #1 attached to #80000001 circuit... 81 | Stream #1: connecting to ('facebookcorewwwi.onion', 443) 82 | Extending #80000001 circuit for hidden service facebookcorewwwi.onion... 83 | Rendezvous established (CellRelayRendezvousEstablished()) 84 | Iterate over responsible dirs of the hidden service 85 | Iterate over introduction points of the hidden service 86 | Create circuit for hsdir 87 | Creating new circuit #80000002 with 185.2.31.8:443 (cx10TorServer; Tor 0.4.0.5) router... 88 | Building 0 hops circuit... 89 | Extending the circuit #80000002 with 132.248.241.5:9001 (toritounam; Tor 0.3.5.8)... 90 | Creating stream #2 attached to #80000002 circuit... 91 | Stream #2: connecting to hsdir 92 | Stream #2: closing... 93 | Destroy circuit #80000002 94 | Creating new circuit #80000003 with 185.2.31.8:443 (cx10TorServer; Tor 0.4.0.5) router... 95 | Building 0 hops circuit... 96 | Extending the circuit #80000003 with 88.198.17.248:8443 (bauruine31; Tor 0.4.1.5)... 97 | Introduced (CellRelayIntroduceAck()) 98 | Destroy circuit #80000003 99 | Creating stream #3 attached to #80000001 circuit... 100 | Stream #3: connecting to ('www.facebookcorewwwi.onion', 443) 101 | Extending #80000001 circuit for hidden service facebookcorewwwi.onion... 102 | Response status: 200 103 | Writing to file index.html 104 | Stream #1: closing... 105 | Stream #3: closing... 106 | Closing guard connections... 107 | Destroy circuit #80000001 108 | ``` 109 | 110 | Usage examples 111 | ----------- 112 | 113 | A basic example of how to send some data to a clearnet host or a hidden service: 114 | ```python 115 | from torpy import TorClient 116 | 117 | hostname = 'ifconfig.me' # It's possible use onion hostname here as well 118 | with TorClient() as tor: 119 | # Choose random guard node and create 3-hops circuit 120 | with tor.create_circuit(3) as circuit: 121 | # Create tor stream to host 122 | with circuit.create_stream((hostname, 80)) as stream: 123 | # Now we can communicate with host 124 | stream.send(b'GET / HTTP/1.0\r\nHost: %s\r\n\r\n' % hostname.encode()) 125 | recv = stream.recv(1024) 126 | ``` 127 | 128 | TorHttpAdapter is a convenient Tor adapter for the [requests library](https://2.python-requests.org/en/master/user/advanced/#transport-adapters). 129 | The following example shows the usage of TorHttpAdapter for multi-threaded HTTP requests: 130 | ```python 131 | from multiprocessing.pool import ThreadPool 132 | from torpy.http.requests import tor_requests_session 133 | 134 | with tor_requests_session() as s: # returns requests.Session() object 135 | links = ['http://nzxj65x32vh2fkhk.onion', 'http://facebookcorewwwi.onion'] * 2 136 | 137 | with ThreadPool(3) as pool: 138 | pool.map(s.get, links) 139 | 140 | ``` 141 | 142 | For more examples see [test_integration.py](https://github.com/torpyorg/torpy/blob/master/tests/integration/test_integration.py) 143 | 144 | 145 | Installation 146 | ------------ 147 | * Just `pip3 install torpy` 148 | * Or for using TorHttpAdapter with requests library you need install extras: 149 | `pip3 install torpy[requests]` 150 | 151 | Contribute 152 | ---------- 153 | * Use It 154 | * Code review is appreciated 155 | * Open [Issue], send [PR] 156 | 157 | 158 | TODO 159 | ---- 160 | - [ ] Implement v3 hidden services [specification](https://gitweb.torproject.org/torspec.git/tree/rend-spec-v3.txt) 161 | - [ ] Refactor Tor cells serialization/deserialization 162 | - [ ] More unit tests 163 | - [ ] Rewrite the library using asyncio 164 | - [ ] Implement onion services 165 | 166 | 167 | License 168 | ------- 169 | Licensed under the Apache License, Version 2.0 170 | 171 | 172 | References 173 | ---------- 174 | - Official [Tor](https://gitweb.torproject.org/tor.git/) client 175 | - [Pycepa](https://github.com/pycepa/pycepa) 176 | - [TorPylle](https://github.com/cea-sec/TorPylle) 177 | - [TinyTor](https://github.com/Marten4n6/TinyTor) 178 | - C++ Windows only implementation [Mini-tor](https://github.com/wbenny/mini-tor) 179 | - Nice Java implementation [Orchid](https://github.com/subgraph/Orchid) 180 | 181 | 182 | [Python Versions]: https://img.shields.io/badge/python-3.6,%203.7,%203.8,%203.9-blue.svg 183 | [Issue]: https://github.com/torpyorg/torpy/issues 184 | [PR]: https://github.com/torpyorg/torpy/pulls -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | --- 2 | environment: 3 | matrix: 4 | - TOXENV: py36-unit 5 | - TOXENV: py36-integration 6 | - TOXENV: py37-unit 7 | - TOXENV: py37-integration 8 | 9 | build: off 10 | 11 | install: 12 | - pip install tox 13 | 14 | test_script: 15 | - tox -v 16 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | log_cli=true 3 | log_level=NOTSET 4 | log_format=[%(asctime)s] [%(threadName)-16s] %(message)s -------------------------------------------------------------------------------- /requirements-flake8.txt: -------------------------------------------------------------------------------- 1 | flake8 2 | flake8-colors 3 | flake8-quotes 4 | flake8-docstrings>=0.2.7 5 | pydocstyle<4 6 | flake8-import-order>=0.9 7 | flake8-typing-imports>=1.1 8 | pep8-naming 9 | -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | coverage 3 | pytest-cov -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile setup.py 6 | # 7 | cffi==1.14.4 # via cryptography 8 | cryptography==3.4.4 9 | pycparser==2.20 # via cffi 10 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | license_files = LICENSE.txt -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | from io import open 17 | from os import path 18 | 19 | from setuptools import setup, find_packages 20 | 21 | here = path.abspath(path.dirname(__file__)) 22 | 23 | with open(path.join(here, 'README.md'), encoding='utf-8') as f: 24 | long_description = f.read() 25 | 26 | setup( 27 | name='torpy', 28 | version='1.1.6', 29 | description='Pure python tor protocol implementation', 30 | long_description=long_description, 31 | long_description_content_type='text/markdown', 32 | url='https://github.com/torpyorg/torpy', 33 | author='James Brown', 34 | classifiers=[ 35 | 'Development Status :: 4 - Beta', 36 | 'Intended Audience :: Developers', 37 | 'License :: OSI Approved :: Apache Software License', 38 | 'Programming Language :: Python :: 3.6', 39 | 'Programming Language :: Python :: 3.7', 40 | 'Programming Language :: Python :: 3.8', 41 | 'Programming Language :: Python :: 3.9', 42 | ], 43 | keywords='python proxy anonymity privacy socks tor protocol onion hiddenservice', 44 | packages=find_packages(exclude=['tests']), 45 | python_requires='>=3.6', 46 | install_requires=['cryptography>=3.2'], 47 | extras_require={'requests': 'requests>2.9,!=2.17.0,!=2.18.0'}, 48 | entry_points={'console_scripts': ['torpy_cli=torpy.cli.console:main', 49 | 'torpy_socks=torpy.cli.socks:main'] 50 | }, 51 | project_urls={ 52 | 'Bug Reports': 'https://github.com/torpyorg/torpy/issues', 53 | 'Source': 'https://github.com/torpyorg/torpy/', 54 | }, 55 | ) 56 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torpyorg/torpy/ebf000cc93d2d7b3ddf68f2604afd46c998ae1c1/tests/__init__.py -------------------------------------------------------------------------------- /tests/integration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torpyorg/torpy/ebf000cc93d2d7b3ddf68f2604afd46c998ae1c1/tests/integration/__init__.py -------------------------------------------------------------------------------- /tests/integration/test_integration.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import os 17 | import socket 18 | import logging 19 | from threading import Event 20 | from selectors import EVENT_READ 21 | from multiprocessing.pool import ThreadPool 22 | 23 | import requests 24 | 25 | from torpy import TorClient 26 | from torpy.stream import TorStream 27 | from torpy.utils import AuthType, recv_all, retry 28 | from torpy.http.adapter import TorHttpAdapter 29 | from torpy.hiddenservice import HiddenService 30 | from torpy.http.requests import TorRequests, tor_requests_session, do_request as requests_request 31 | from torpy.http.urlopener import do_request as urllib_request 32 | 33 | logging.getLogger('requests').setLevel(logging.CRITICAL) 34 | logging.basicConfig(format='[%(asctime)s] [%(threadName)-16s] %(message)s', level=logging.DEBUG) 35 | logger = logging.getLogger(__name__) 36 | 37 | 38 | HS_BASIC_HOST = os.getenv('HS_BASIC_HOST') 39 | HS_BASIC_AUTH = os.getenv('HS_BASIC_AUTH') 40 | 41 | HS_STEALTH_HOST = os.getenv('HS_STEALTH_HOST') 42 | HS_STEALTH_AUTH = os.getenv('HS_STEALTH_AUTH') 43 | 44 | RETRIES = 3 45 | 46 | 47 | @retry(RETRIES, (TimeoutError, ConnectionError, )) 48 | def test_clearnet_raw(): 49 | hostname = 'ifconfig.me' 50 | with TorClient() as tor: 51 | # Choose random guard node and create 3-hops circuit 52 | with tor.create_circuit(3) as circuit: 53 | # Create tor stream to host 54 | with circuit.create_stream((hostname, 80)) as stream: 55 | # Send some data to it 56 | stream.send(b'GET / HTTP/1.0\r\nHost: %s\r\n\r\n' % hostname.encode()) 57 | recv = recv_all(stream).decode() 58 | logger.warning('recv: %s', recv) 59 | search_ip = '.'.join(circuit.last_node.router.ip.split('.')[:-1]) + '.' 60 | assert search_ip in recv, 'wrong data received' 61 | 62 | 63 | @retry(RETRIES, (TimeoutError, ConnectionError, )) 64 | def test_onion_raw(): 65 | hostname = 'nzxj65x32vh2fkhk.onion' 66 | with TorClient() as tor: 67 | # Choose random guard node and create 3-hops circuit 68 | with tor.create_circuit(3) as circuit: 69 | # Create tor stream to host 70 | with circuit.create_stream((hostname, 80)) as stream: 71 | # Send some data to it 72 | stream.send(b'GET / HTTP/1.0\r\nHost: %s\r\n\r\n' % hostname.encode()) 73 | 74 | recv = recv_all(stream).decode() 75 | logger.warning('recv: %s', recv) 76 | assert 'StickyNotes' in recv, 'wrong data received' 77 | 78 | 79 | def test_requests_no_agent(): 80 | data = requests_request('https://httpbin.org/headers', retries=RETRIES) 81 | assert 'User-Agent' not in data 82 | 83 | 84 | def test_requests(): 85 | data = requests_request('https://httpbin.org/headers', headers={'User-Agent': 'Mozilla/5.0'}, retries=RETRIES) 86 | assert 'Mozilla' in data 87 | 88 | 89 | def test_requests_session(): 90 | with TorClient() as tor: 91 | with tor.get_guard() as guard: 92 | adapter = TorHttpAdapter(guard, 3, retries=RETRIES) 93 | 94 | with requests.Session() as s: 95 | s.headers.update({'User-Agent': 'Mozilla/5.0'}) 96 | s.mount('http://', adapter) 97 | s.mount('https://', adapter) 98 | 99 | r = s.get('https://google.com', timeout=30) 100 | logger.warning(r) 101 | logger.warning(r.text) 102 | assert r.text.rstrip().endswith('') 103 | 104 | r = s.get('https://stackoverflow.com/questions/tagged/python') 105 | assert r.text.rstrip().endswith('') 106 | logger.warning(r) 107 | logger.warning(r.text) 108 | 109 | 110 | def test_urlopener_no_agent(): 111 | data = urllib_request('https://httpbin.org/headers', verbose=1, retries=RETRIES) 112 | assert 'User-Agent' not in data 113 | 114 | 115 | def test_urlopener(): 116 | data = urllib_request('https://httpbin.org/headers', headers=[('User-Agent', 'Mozilla/5.0')], verbose=1, 117 | retries=RETRIES) 118 | assert 'Mozilla' in data 119 | 120 | 121 | def test_multi_threaded(): 122 | auth_data = {HS_BASIC_HOST: (HS_BASIC_AUTH, AuthType.Basic)} if HS_BASIC_HOST and HS_BASIC_AUTH else None 123 | 124 | with TorRequests(auth_data=auth_data) as tor_requests: 125 | links = [ 126 | 'https://httpbin.org/headers', 127 | 'https://google.com', 128 | 'https://ifconfig.me', 129 | 'http://facebookcorewwwi.onion', 130 | ] 131 | if HS_BASIC_HOST: 132 | links.append('http://' + HS_BASIC_HOST) 133 | links = links * 10 134 | 135 | with tor_requests.get_session(retries=RETRIES) as sess: 136 | 137 | def process(link): 138 | try: 139 | logger.debug('get link: %s', link) 140 | r = sess.get(link, timeout=30) 141 | logger.warning('get link %s finish: %s', link, r) 142 | return r 143 | except BaseException: 144 | logger.exception('get link %s error', link) 145 | 146 | pool = ThreadPool(10) 147 | for i, w in enumerate(pool._pool): 148 | w.name = 'Worker{}'.format(i) 149 | results = pool.map(process, links) 150 | pool.close() 151 | pool.join() 152 | logger.debug('test_multi_threaded ends: %r', results) 153 | 154 | 155 | @retry(RETRIES, (TimeoutError, ConnectionError, )) 156 | def test_basic_auth(): 157 | """Connecting to Hidden Service with 'Basic' authorization.""" 158 | if not HS_BASIC_HOST or not HS_BASIC_AUTH: 159 | logger.warning('Skip test_basic_auth()') 160 | return 161 | 162 | hs = HiddenService(HS_BASIC_HOST, HS_BASIC_AUTH, AuthType.Basic) 163 | with TorClient() as tor: 164 | # Choose random guard node and create 3-hops circuit 165 | with tor.create_circuit(3) as circuit: 166 | # Create tor stream to host 167 | with circuit.create_stream((hs, 80)) as stream: 168 | # Send some data to it 169 | stream.send(b'GET / HTTP/1.0\r\nHost: %s.onion\r\n\r\n' % hs.onion.encode()) 170 | recv = recv_all(stream).decode() 171 | logger.warning('recv: %s', recv) 172 | 173 | 174 | @retry(RETRIES, (TimeoutError, ConnectionError, )) 175 | def test_stealth_auth(): 176 | """Connecting to Hidden Service with 'Stealth' authorization.""" 177 | if not HS_STEALTH_HOST or not HS_STEALTH_AUTH: 178 | logger.warning('Skip test_stealth_auth()') 179 | return 180 | 181 | hs = HiddenService(HS_STEALTH_HOST, HS_STEALTH_AUTH, AuthType.Stealth) 182 | with TorClient() as tor: 183 | # Choose random guard node and create 3-hops circuit 184 | with tor.create_circuit(3) as circuit: 185 | # Create tor stream to host 186 | with circuit.create_stream((hs, 80)) as stream: 187 | # Send some data to it 188 | stream.send(b'GET / HTTP/1.0\r\nHost: %s\r\n\r\n' % hs.hostname.encode()) 189 | recv = recv_all(stream).decode() 190 | logger.warning('recv: %s', recv) 191 | 192 | 193 | @retry(RETRIES, (TimeoutError, ConnectionError, )) 194 | def test_basic_auth_pre(): 195 | """Using pre-defined authorization data for making HTTP requests.""" 196 | if not HS_BASIC_HOST or not HS_BASIC_AUTH: 197 | logger.warning('Skip test_basic_auth()') 198 | return 199 | 200 | hidden_service = HS_BASIC_HOST 201 | auth_data = {HS_BASIC_HOST: (HS_BASIC_AUTH, AuthType.Basic)} 202 | with TorClient(auth_data=auth_data) as tor: 203 | # Choose random guard node and create 3-hops circuit 204 | with tor.create_circuit(3) as circuit: 205 | # Create tor stream to host 206 | with circuit.create_stream((hidden_service, 80)) as stream: 207 | # Send some data to it 208 | stream.send(b'GET / HTTP/1.0\r\nHost: %s.onion\r\n\r\n' % hidden_service.encode()) 209 | recv = recv_all(stream).decode() 210 | logger.warning('recv: %s', recv) 211 | 212 | 213 | def test_requests_hidden(): 214 | """Using pre-defined authorization data for making HTTP requests by tor_requests_session.""" 215 | if not HS_BASIC_HOST or not HS_BASIC_AUTH: 216 | logger.warning('Skip test_requests_hidden()') 217 | return 218 | 219 | auth_data = {HS_BASIC_HOST: (HS_BASIC_AUTH, AuthType.Basic)} 220 | with tor_requests_session(auth_data=auth_data, retries=RETRIES) as sess: 221 | r = sess.get('http://{}/'.format(HS_BASIC_HOST), timeout=30) 222 | logger.warning(r) 223 | logger.warning(r.text) 224 | 225 | 226 | @retry(2, (TimeoutError, ConnectionError, )) 227 | def test_select(): 228 | sock_r, sock_w = socket.socketpair() 229 | 230 | events = {TorStream: {'data': Event(), 'close': Event()}, 231 | socket.socket: {'data': Event(), 'close': Event()}} 232 | 233 | hostname = 'ifconfig.me' 234 | with TorClient() as tor: 235 | with tor.get_guard() as guard: 236 | 237 | def recv_callback(sock_or_stream, mask): 238 | logger.debug(f'recv_callback {sock_or_stream}') 239 | kind = type(sock_or_stream) 240 | data = sock_or_stream.recv(1024) 241 | logger.info('%s: %r', kind.__name__, data.decode()) 242 | if data: 243 | events[kind]['data'].set() 244 | else: 245 | logger.debug('closing') 246 | guard.unregister(sock_or_stream) 247 | events[kind]['close'].set() 248 | 249 | with guard.create_circuit(3) as circuit: 250 | with circuit.create_stream((hostname, 80)) as stream: 251 | guard.register(sock_r, EVENT_READ, recv_callback) 252 | guard.register(stream, EVENT_READ, recv_callback) 253 | 254 | stream.send(b'GET / HTTP/1.0\r\nHost: %s\r\n\r\n' % hostname.encode()) 255 | sock_w.send(b'some data') 256 | 257 | assert events[socket.socket]['data'].wait(10), 'no sock data received' 258 | assert events[TorStream]['data'].wait(30), 'no stream data received' 259 | 260 | sock_w.close() 261 | assert events[socket.socket]['close'].wait(10), 'no sock close received' 262 | assert events[TorStream]['close'].wait(10), 'no stream close received' 263 | -------------------------------------------------------------------------------- /tests/unittest/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torpyorg/torpy/ebf000cc93d2d7b3ddf68f2604afd46c998ae1c1/tests/unittest/__init__.py -------------------------------------------------------------------------------- /tests/unittest/data/dir_certs/dir_cert_real: -------------------------------------------------------------------------------- 1 | dir-key-certificate-version 3 2 | fingerprint 14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 3 | dir-key-published 2019-06-01 00:00:00 4 | dir-key-expires 2019-11-01 00:00:00 5 | dir-identity-key 6 | -----BEGIN RSA PUBLIC KEY----- 7 | MIIBigKCAYEA7cZXvDRxfjDYtr9/9UsQ852+6cmHMr8VVh8GkLwbq3RzqjkULwQ2 8 | R9mFvG4FnqMcMKXi62rYYA3fZL1afhT804cpvyp/D3dPM8QxW88fafFAgIFP4LiD 9 | 0JYjnF8cva5qZ0nzlWnMXLb32IXSvsGSE2FRyAV0YN9a6k967LSgCfUnZ+IKMezW 10 | 1vhL9YK4QIfsDowgtVsavg63GzGmA7JvZmn77+/J5wKz11vGr7Wttf8XABbH2taX 11 | O9j/KGBOX2OKhoF3mXfZSmUO2dV9NMwtkJ7zD///Ny6sfApWV6kVP4O9TdG3bAsl 12 | +fHCoCKgF/jAAWzh6VckQTOPzQZaH5aMWfXrDlzFWg17MjonI+bBTD2Ex2pHczzJ 13 | bN7coDMRH2SuOXv8wFf27KdUxZ/GcrXSRGzlRLygxqlripUanjVGN2JvrVQVr0kz 14 | pjNjiZl2z8ZyZ5d4zQuBi074JPGgx62xAstP37v1mPw14sIWfLgY16ewYuS5bCxV 15 | lyS28jsPht9VAgMBAAE= 16 | -----END RSA PUBLIC KEY----- 17 | dir-signing-key 18 | -----BEGIN RSA PUBLIC KEY----- 19 | MIIBigKCAYEAuMM87vcfVHbSaAK3oWvwNCxZ8fW+W5PM2hNWOyGaXF418FmOf863 20 | GW03l9wKvRPYKe0/wPaobkHZboK8rL1iSDx4CaK9EyKg7updiaKMI9Ml2XDCLYzL 21 | bRf4vlZkodgFaIBgKzhQwxq0f+yT7sbjToHD38WlE8lzCjjf+GEnSsfYJB05C083 22 | iJgKLQFHiwTt2GTR2P0oknSYN+UhFXex5EcsbcLL6dIRPONrYLEpVpKuH6YAHwJk 23 | rGyA1GdZ0QlO6tlYSJ4X3DnJPisOF/s9TB9K9GeSQqKghXHnYEDepPjgIjNvd1Ne 24 | bi0JmY0Z/qsc22E65cbYWon2g5mQCRb+MhIsQWIpQZpGqiPQ4SmHLYdiX+iHP1WS 25 | XBg9sii7aDAFUY94XQJtP8Jfg/p+kBKbw1CkCp8d/D5VdCVBvo+zZ3JD7/9fYbls 26 | P3ZjH1LhxX1ENEuwBCk8DQ3aaJiDRCLHhxsXM0B5afQhyk2AJuYiqzdKcxw527UC 27 | Rx6DW1SvY/DfAgMBAAE= 28 | -----END RSA PUBLIC KEY----- 29 | dir-key-crosscert 30 | -----BEGIN ID SIGNATURE----- 31 | W4M9vToRHli3Xhf0nzQs9T5eCs7o91c64Tkr3aiHIzyye22QsucSAErz3AvNnxfP 32 | rW3EoH76U7hdkq+v29DznRXDd1f5Ksfu1G1ZlxuQmkj2cw2eoXanjzf+JgqZ5uno 33 | AKWBeu9j05PowXCwgy5c6ok6wghCnXf3Kjng5mtBHgM9BThks8XTrbD+coi/RQwb 34 | 8lUCdy98632cV21ydo99QBENbZvSShKZ8sfphhjI5pYbRpZyT6klKuV1+MAPOV75 35 | 3amRfkXpo1q1X6Dtg+NTzknjqILTt9lXJOZvLF59aRprT2eDCwN6ra9njGpq1jOV 36 | tjKI51LLd0R8DO0ujDR/CzXr9XBZS/S/Jr1F1yIdSdGWNSEECsCZ9HajjR0Y3LJT 37 | fMNOT+kNJFyAuMcDP7ltLidoxt+1TJt5HYlmkMicwCYyKGpKtr2Q+OMFBlz/5jqR 38 | Gh2IZkqF/UTLu+dwgKzu3V/JOht6nW10ROcA+CYs1sVpQfpxNBkXbmtCbcfFVDhV 39 | -----END ID SIGNATURE----- 40 | dir-key-certification 41 | -----BEGIN SIGNATURE----- 42 | aI92/sOUwiZ8pxXOZu3iuwJ4rFhZIwfznTm+Pc9Q0uL5Zv7VyPJ56cMNYwGtTMGU 43 | WrVYk2h5lFst1/v0extmgFCIjPc6+bxohcg3E1opHL0PPjS/jHPC7i2rKufitrxO 44 | 5IjQKtUoFkPE3RfJ2YgF6X3AAZCf4XHJX6t6RQBTtfyVk4N8GA2hK3p4sVdIb9yo 45 | WQd8pEtDb6UaM3H/xz7XBfwJfjJUriPJPj0UOn3NGjWG4htwgn26I8iOXn43aovD 46 | 9gLjT8XYSmnBrcsyVPoRuLrkAUya9j7IJauScXb0lC5ff/r+R0sRKacHz/C2mRNz 47 | 0MUOZzclXGYgoEI5SlVmgjExAk1lzutHrGTU2i7gOHSyLUBkkuwBwwGRvdwin6MA 48 | dqds2KuCHuHgY+mS8cNKO2sJAFYsMZF3GgK5IhBXCeNpFWHdgsAmIvHQjKTj2BKJ 49 | LgYpBddfqew0nZauSLdutpBFXngdD9QXW2YIzCxb6pQDqBaYOygEoHQqVMWVn0BA 50 | -----END SIGNATURE----- -------------------------------------------------------------------------------- /tests/unittest/data/network_status/consensus: -------------------------------------------------------------------------------- 1 | network-status-version 3 2 | vote-status consensus 3 | consensus-method 26 4 | valid-after 2017-05-25 04:46:30 5 | fresh-until 2017-05-25 04:46:40 6 | valid-until 2017-05-25 04:46:50 7 | voting-delay 2 2 8 | client-versions 9 | server-versions 10 | known-flags Authority Exit Fast Guard HSDir NoEdConsensus Running Stable V2Dir Valid 11 | recommended-client-protocols Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2 12 | recommended-relay-protocols Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2 13 | required-client-protocols Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2 14 | required-relay-protocols Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=3-4 LinkAuth=1 Microdesc=1 Relay=1-2 15 | dir-source test001a 596CD48D61FDA4E868F4AA10FF559917BE3B1A35 127.0.0.1 127.0.0.1 7001 5001 16 | contact auth1@test.test 17 | vote-digest 2E7177224BBA39B505F7608FF376C07884CF926F 18 | dir-source test000a BCB380A633592C218757BEE11E630511A485658A 127.0.0.1 127.0.0.1 7000 5000 19 | contact auth0@test.test 20 | vote-digest 5DD41617166FFB82882A117EEFDA0353A2794DC5 21 | r test002r NIIl+DyFR5ay3WNk5lyxibM71pY UzQp+EE8G0YCKtNlZVy+3h5tv0Q 2017-05-25 04:46:11 127.0.0.1 5002 7002 22 | s Exit Fast Guard HSDir Running Stable V2Dir Valid 23 | v Tor 0.3.0.7 24 | pr Cons=1-2 Desc=1-2 DirCache=1 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 25 | w Bandwidth=0 Unmeasured=1 26 | p accept 1-65535 27 | r test001a qgzRpIKSW809FnL4tntRtWgOiwo x8yR5mi/DBbLg46qwGQ96Dno+nc 2017-05-25 04:46:12 127.0.0.1 5001 7001 28 | s Authority Exit Fast Guard HSDir Running V2Dir Valid 29 | v Tor 0.3.0.7 30 | pr Cons=1-2 Desc=1-2 DirCache=1 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 31 | w Bandwidth=0 Unmeasured=1 32 | p reject 1-65535 33 | r test000a 3nJC+LvtNmx6kw23x1WE90pyIj4 Hg3NyPqDZoRQN8hVI5Vi6B+pofw 2017-05-25 04:46:12 127.0.0.1 5000 7000 34 | s Authority Exit Fast Guard HSDir Running Stable V2Dir Valid 35 | v Tor 0.3.0.7 36 | pr Cons=1-2 Desc=1-2 DirCache=1 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 37 | w Bandwidth=0 Unmeasured=1 38 | p reject 1-65535 39 | directory-footer 40 | bandwidth-weights Wbd=3333 Wbe=0 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=3333 Wee=10000 Weg=3333 Wem=10000 Wgb=10000 Wgd=3333 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=3333 Wme=0 Wmg=0 Wmm=10000 41 | directory-signature 596CD48D61FDA4E868F4AA10FF559917BE3B1A35 9FBF54D6A62364320308A615BF4CF6B27B254FAD 42 | -----BEGIN SIGNATURE----- 43 | Ho0rLojfLHs9cSPFxe6znuGuFU8BvRr6gnH1gULTjUZO0NSQvo5N628KFeAsq+pT 44 | ElieQeV6UfwnYN1U2tomhBYv3+/p1xBxYS5oTDAITxLUYvH4pLYz09VutwFlFFtU 45 | r/satajuOMST0M3wCCBC4Ru5o5FSklwJTPJ/tWRXDCEHv/N5ZUUkpnNdn+7tFSZ9 46 | eFrPxPcQvB05BESo7C4/+ZnZVO/wduObSYu04eWwTEog2gkSWmsztKoXpx1QGrtG 47 | sNL22Ws9ySGDO/ykFFyxkcuyB5A8oPyedR7DrJUfCUYyB8o+XLNwODkCFxlmtFOj 48 | ci356fosgLiM1sVqCUkNdA== 49 | -----END SIGNATURE----- 50 | directory-signature BCB380A633592C218757BEE11E630511A485658A 9CA027E05B0CE1500D90DA13FFDA8EDDCD40A734 51 | -----BEGIN SIGNATURE----- 52 | uiAt8Ir27pYFX5fNKiVZDoa6ELVEtg/E3YeYHAnlSSRzpacLMMTN/HhF//Zvv8Zj 53 | FKT95v77xKvE6b8s7JjB3ep6coiW4tkLqjDiONG6iDRKBmy6D+RZgf1NMxl3gWaZ 54 | ShINORJMW9nglnBbysP7egPiX49w1igVZQLM1C2ppphK6uO5EGcK6nDJF4LVDJ7B 55 | Fvt2yhY+gsiG3oSrhsP0snQnFfvEeUFO/r2gRVJ1FoMXUttaOCtmj268xS08eZ0m 56 | MS+u6gHEM1dYkwpA+LzE9G4akPRhvRjMDaF0RMuLQ7pY5v44uE5OX5n/GKWRgzVZ 57 | DH+ubl6BuqpQxYQXaHZ5iw== 58 | -----END SIGNATURE----- 59 | -------------------------------------------------------------------------------- /tests/unittest/data/network_status/consensus.json: -------------------------------------------------------------------------------- 1 | { 2 | "network_status_version": 3, 3 | "vote_status": "consensus", 4 | "consensus_method": 26, 5 | "valid_after": { 6 | "_isoformat": "2017-05-25T04:46:30" 7 | }, 8 | "fresh_until": { 9 | "_isoformat": "2017-05-25T04:46:40" 10 | }, 11 | "valid_until": { 12 | "_isoformat": "2017-05-25T04:46:50" 13 | }, 14 | "voting_delay": "2 2", 15 | "client_versions": "", 16 | "server_versions": "", 17 | "known_flags": "Authority Exit Fast Guard HSDir NoEdConsensus Running Stable V2Dir Valid", 18 | "recommended_client_protocols": "Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2", 19 | "recommended_relay_protocols": "Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2", 20 | "required_client_protocols": "Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2", 21 | "required_relay_protocols": "Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=3-4 LinkAuth=1 Microdesc=1 Relay=1-2", 22 | "voters": [ 23 | { 24 | "nickname": "test001a", 25 | "fingerprint": "596CD48D61FDA4E868F4AA10FF559917BE3B1A35", 26 | "hostname": "127.0.0.1", 27 | "address": "127.0.0.1", 28 | "dir_port": "7001", 29 | "or_port": "5001", 30 | "contact": "auth1@test.test", 31 | "vote_digest": "2E7177224BBA39B505F7608FF376C07884CF926F" 32 | }, 33 | { 34 | "nickname": "test000a", 35 | "fingerprint": "BCB380A633592C218757BEE11E630511A485658A", 36 | "hostname": "127.0.0.1", 37 | "address": "127.0.0.1", 38 | "dir_port": "7000", 39 | "or_port": "5000", 40 | "contact": "auth0@test.test", 41 | "vote_digest": "5DD41617166FFB82882A117EEFDA0353A2794DC5" 42 | } 43 | ], 44 | "routers": [ 45 | { 46 | "_nickname": "test002r", 47 | "_fingerprint": { 48 | "_bytes": "348225f83c854796b2dd6364e65cb189b33bd696" 49 | }, 50 | "_digest": { 51 | "_bytes": "533429f8413c1b46022ad365655cbede1e6dbf44" 52 | }, 53 | "_ip": "127.0.0.1", 54 | "_or_port": 5002, 55 | "_dir_port": 7002, 56 | "_version": "Tor 0.3.0.7", 57 | "_flags": [ 58 | { 59 | "_enum": "RouterFlags.Exit" 60 | }, 61 | { 62 | "_enum": "RouterFlags.Fast" 63 | }, 64 | { 65 | "_enum": "RouterFlags.Guard" 66 | }, 67 | { 68 | "_enum": "RouterFlags.HSDir" 69 | }, 70 | { 71 | "_enum": "RouterFlags.Running" 72 | }, 73 | { 74 | "_enum": "RouterFlags.Stable" 75 | }, 76 | { 77 | "_enum": "RouterFlags.V2Dir" 78 | }, 79 | { 80 | "_enum": "RouterFlags.Valid" 81 | } 82 | ] 83 | }, 84 | { 85 | "_nickname": "test001a", 86 | "_fingerprint": { 87 | "_bytes": "aa0cd1a482925bcd3d1672f8b67b51b5680e8b0a" 88 | }, 89 | "_digest": { 90 | "_bytes": "c7cc91e668bf0c16cb838eaac0643de839e8fa77" 91 | }, 92 | "_ip": "127.0.0.1", 93 | "_or_port": 5001, 94 | "_dir_port": 7001, 95 | "_version": "Tor 0.3.0.7", 96 | "_flags": [ 97 | { 98 | "_enum": "RouterFlags.Authority" 99 | }, 100 | { 101 | "_enum": "RouterFlags.Exit" 102 | }, 103 | { 104 | "_enum": "RouterFlags.Fast" 105 | }, 106 | { 107 | "_enum": "RouterFlags.Guard" 108 | }, 109 | { 110 | "_enum": "RouterFlags.HSDir" 111 | }, 112 | { 113 | "_enum": "RouterFlags.Running" 114 | }, 115 | { 116 | "_enum": "RouterFlags.V2Dir" 117 | }, 118 | { 119 | "_enum": "RouterFlags.Valid" 120 | } 121 | ] 122 | }, 123 | { 124 | "_nickname": "test000a", 125 | "_fingerprint": { 126 | "_bytes": "de7242f8bbed366c7a930db7c75584f74a72223e" 127 | }, 128 | "_digest": { 129 | "_bytes": "1e0dcdc8fa8366845037c855239562e81fa9a1fc" 130 | }, 131 | "_ip": "127.0.0.1", 132 | "_or_port": 5000, 133 | "_dir_port": 7000, 134 | "_version": "Tor 0.3.0.7", 135 | "_flags": [ 136 | { 137 | "_enum": "RouterFlags.Authority" 138 | }, 139 | { 140 | "_enum": "RouterFlags.Exit" 141 | }, 142 | { 143 | "_enum": "RouterFlags.Fast" 144 | }, 145 | { 146 | "_enum": "RouterFlags.Guard" 147 | }, 148 | { 149 | "_enum": "RouterFlags.HSDir" 150 | }, 151 | { 152 | "_enum": "RouterFlags.Running" 153 | }, 154 | { 155 | "_enum": "RouterFlags.Stable" 156 | }, 157 | { 158 | "_enum": "RouterFlags.V2Dir" 159 | }, 160 | { 161 | "_enum": "RouterFlags.Valid" 162 | } 163 | ] 164 | } 165 | ], 166 | "signatures": [ 167 | { 168 | "algorithm": "sha1", 169 | "identity": "596CD48D61FDA4E868F4AA10FF559917BE3B1A35", 170 | "signing_key_digest": "9FBF54D6A62364320308A615BF4CF6B27B254FAD", 171 | "signature": { 172 | "_bytes": "1e8d2b2e88df2c7b3d7123c5c5eeb39ee1ae154f01bd1afa8271f58142d38d464ed0d490be8e4deb6f0a15e02cabea5312589e41e57a51fc2760dd54dada2684162fdfefe9d71071612e684c30084f12d462f1f8a4b633d3d56eb70165145b54affb1ab5a8ee38c493d0cdf0082042e11bb9a39152925c094cf27fb564570c2107bff379654524a6735d9feeed15267d785acfc4f710bc1d390444a8ec2e3ff999d954eff076e39b498bb4e1e5b04c4a20da09125a6b33b4aa17a71d501abb46b0d2f6d96b3dc921833bfca4145cb191cbb207903ca0fc9e751ec3ac951f09463207ca3e5cb370383902171966b453a3722df9e9fa2c80b88cd6c56a09490d74" 173 | } 174 | }, 175 | { 176 | "algorithm": "sha1", 177 | "identity": "BCB380A633592C218757BEE11E630511A485658A", 178 | "signing_key_digest": "9CA027E05B0CE1500D90DA13FFDA8EDDCD40A734", 179 | "signature": { 180 | "_bytes": "ba202df08af6ee96055f97cd2a25590e86ba10b544b60fc4dd87981c09e5492473a5a70b30c4cdfc7845fff66fbfc66314a4fde6fefbc4abc4e9bf2cec98c1ddea7a728896e2d90baa30e238d1ba88344a066cba0fe45981fd4d3319778166994a120d39124c5bd9e096705bcac3fb7a03e25f8f70d628156502ccd42da9a6984aeae3b910670aea70c91782d50c9ec116fb76ca163e82c886de84ab86c3f4b2742715fbc479414efebda045527516831752db5a382b668f6ebcc52d3c799d26312faeea01c4335758930a40f8bcc4f46e1a90f461bd18cc0da17444cb8b43ba58e6fe38b84e4e5f99ff18a5918335590c7fae6e5e81baaa50c584176876798b" 181 | } 182 | } 183 | ] 184 | } -------------------------------------------------------------------------------- /tests/unittest/data/network_status/consensus_extra: -------------------------------------------------------------------------------- 1 | network-status-version 3 5 2 | vote-status consensus 3 | consensus-method 26 4 | valid-after 2017-05-25 04:46:30 strange 5 | fresh-until 2017-05-25 04:46:40 6 | valid-until 2017-05-25 04:46:50 7 | voting-delay 2 2 8 | client-versions 9 | server-versions 10 | known-flags Authority Exit Fast Guard NewFlag1 HSDir NoEdConsensus Running Stable V2Dir Valid NewFlag2 11 | recommended-client-protocols Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2 12 | recommended-relay-protocols Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2 13 | required-client-protocols Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2 14 | required-relay-protocols Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=3-4 LinkAuth=1 Microdesc=1 Relay=1-2 15 | dir-source test001a 596CD48D61FDA4E868F4AA10FF559917BE3B1A35 127.0.0.1 127.0.0.1 7001 5001 extra_arg1 extra_arg2 16 | contact auth1@test.test 17 | extra-keyword 111 222 18 | vote-digest 2E7177224BBA39B505F7608FF376C07884CF926F AAAAAAA 19 | dir-source test000a BCB380A633592C218757BEE11E630511A485658A 127.0.0.1 127.0.0.1 7000 5000 20 | contact auth0@test.test 21 | vote-digest 5DD41617166FFB82882A117EEFDA0353A2794DC5 22 | extra-keyword 333 444 23 | r test002r NIIl+DyFR5ay3WNk5lyxibM71pY UzQp+EE8G0YCKtNlZVy+3h5tv0Q 2017-05-25 04:46:11 127.0.0.1 5002 7002 24 | s Exit Fast Guard HSDir Running Stable V2Dir Valid NewFlag 25 | v Tor 0.3.0.7 26 | pr Cons=1-2 Desc=1-2 DirCache=1 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 27 | w Bandwidth=0 Unmeasured=1 Strange=2 28 | p accept 1-65535 29 | r test001a qgzRpIKSW809FnL4tntRtWgOiwo x8yR5mi/DBbLg46qwGQ96Dno+nc 2017-05-25 04:46:12 127.0.0.1 5001 7001 30 | s Authority Exit Fast Guard HSDir Running V2Dir Valid 31 | v Tor 0.3.0.7 32 | pr Cons=1-2 Desc=1-2 DirCache=1 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 33 | w Bandwidth=0 Unmeasured=1 34 | p reject 1-65535 35 | r test000a 3nJC+LvtNmx6kw23x1WE90pyIj4 Hg3NyPqDZoRQN8hVI5Vi6B+pofw 2017-05-25 04:46:12 127.0.0.1 5000 7000 36 | s Authority Exit Fast Guard HSDir Running Stable V2Dir Valid 37 | v Tor 0.3.0.7 38 | pr Cons=1-2 Desc=1-2 DirCache=1 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 39 | w Bandwidth=0 Unmeasured=1 40 | p reject 1-65535 41 | directory-footer 42 | bandwidth-weights Wbd=3333 Wbe=0 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=3333 Wee=10000 Weg=3333 Wem=10000 Wgb=10000 Wgd=3333 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=3333 Wme=0 Wmg=0 Wmm=10000 43 | directory-signature 596CD48D61FDA4E868F4AA10FF559917BE3B1A35 9FBF54D6A62364320308A615BF4CF6B27B254FAD 44 | -----BEGIN SIGNATURE----- 45 | Ho0rLojfLHs9cSPFxe6znuGuFU8BvRr6gnH1gULTjUZO0NSQvo5N628KFeAsq+pT 46 | ElieQeV6UfwnYN1U2tomhBYv3+/p1xBxYS5oTDAITxLUYvH4pLYz09VutwFlFFtU 47 | r/satajuOMST0M3wCCBC4Ru5o5FSklwJTPJ/tWRXDCEHv/N5ZUUkpnNdn+7tFSZ9 48 | eFrPxPcQvB05BESo7C4/+ZnZVO/wduObSYu04eWwTEog2gkSWmsztKoXpx1QGrtG 49 | sNL22Ws9ySGDO/ykFFyxkcuyB5A8oPyedR7DrJUfCUYyB8o+XLNwODkCFxlmtFOj 50 | ci356fosgLiM1sVqCUkNdA== 51 | -----END SIGNATURE----- 52 | directory-signature BCB380A633592C218757BEE11E630511A485658A 9CA027E05B0CE1500D90DA13FFDA8EDDCD40A734 53 | -----BEGIN SIGNATURE----- 54 | uiAt8Ir27pYFX5fNKiVZDoa6ELVEtg/E3YeYHAnlSSRzpacLMMTN/HhF//Zvv8Zj 55 | FKT95v77xKvE6b8s7JjB3ep6coiW4tkLqjDiONG6iDRKBmy6D+RZgf1NMxl3gWaZ 56 | ShINORJMW9nglnBbysP7egPiX49w1igVZQLM1C2ppphK6uO5EGcK6nDJF4LVDJ7B 57 | Fvt2yhY+gsiG3oSrhsP0snQnFfvEeUFO/r2gRVJ1FoMXUttaOCtmj268xS08eZ0m 58 | MS+u6gHEM1dYkwpA+LzE9G4akPRhvRjMDaF0RMuLQ7pY5v44uE5OX5n/GKWRgzVZ 59 | DH+ubl6BuqpQxYQXaHZ5iw== 60 | -----END SIGNATURE----- 61 | -------------------------------------------------------------------------------- /tests/unittest/data/network_status/consensus_extra.json: -------------------------------------------------------------------------------- 1 | { 2 | "network_status_version": 3, 3 | "vote_status": "consensus", 4 | "consensus_method": 26, 5 | "valid_after": { 6 | "_isoformat": "2017-05-25T04:46:30" 7 | }, 8 | "fresh_until": { 9 | "_isoformat": "2017-05-25T04:46:40" 10 | }, 11 | "valid_until": { 12 | "_isoformat": "2017-05-25T04:46:50" 13 | }, 14 | "voting_delay": "2 2", 15 | "client_versions": "", 16 | "server_versions": "", 17 | "known_flags": "Authority Exit Fast Guard NewFlag1 HSDir NoEdConsensus Running Stable V2Dir Valid NewFlag2", 18 | "recommended_client_protocols": "Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2", 19 | "recommended_relay_protocols": "Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2", 20 | "required_client_protocols": "Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2", 21 | "required_relay_protocols": "Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=3-4 LinkAuth=1 Microdesc=1 Relay=1-2", 22 | "voters": [ 23 | { 24 | "nickname": "test001a", 25 | "fingerprint": "596CD48D61FDA4E868F4AA10FF559917BE3B1A35", 26 | "hostname": "127.0.0.1", 27 | "address": "127.0.0.1", 28 | "dir_port": "7001", 29 | "or_port": "5001", 30 | "contact": "auth1@test.test", 31 | "vote_digest": "2E7177224BBA39B505F7608FF376C07884CF926F AAAAAAA" 32 | }, 33 | { 34 | "nickname": "test000a", 35 | "fingerprint": "BCB380A633592C218757BEE11E630511A485658A", 36 | "hostname": "127.0.0.1", 37 | "address": "127.0.0.1", 38 | "dir_port": "7000", 39 | "or_port": "5000", 40 | "contact": "auth0@test.test", 41 | "vote_digest": "5DD41617166FFB82882A117EEFDA0353A2794DC5" 42 | } 43 | ], 44 | "routers": [ 45 | { 46 | "_nickname": "test002r", 47 | "_fingerprint": { 48 | "_bytes": "348225f83c854796b2dd6364e65cb189b33bd696" 49 | }, 50 | "_digest": { 51 | "_bytes": "533429f8413c1b46022ad365655cbede1e6dbf44" 52 | }, 53 | "_ip": "127.0.0.1", 54 | "_or_port": 5002, 55 | "_dir_port": 7002, 56 | "_version": "Tor 0.3.0.7", 57 | "_flags": [ 58 | { 59 | "_enum": "RouterFlags.Exit" 60 | }, 61 | { 62 | "_enum": "RouterFlags.Fast" 63 | }, 64 | { 65 | "_enum": "RouterFlags.Guard" 66 | }, 67 | { 68 | "_enum": "RouterFlags.HSDir" 69 | }, 70 | { 71 | "_enum": "RouterFlags.Running" 72 | }, 73 | { 74 | "_enum": "RouterFlags.Stable" 75 | }, 76 | { 77 | "_enum": "RouterFlags.V2Dir" 78 | }, 79 | { 80 | "_enum": "RouterFlags.Valid" 81 | } 82 | ] 83 | }, 84 | { 85 | "_nickname": "test001a", 86 | "_fingerprint": { 87 | "_bytes": "aa0cd1a482925bcd3d1672f8b67b51b5680e8b0a" 88 | }, 89 | "_digest": { 90 | "_bytes": "c7cc91e668bf0c16cb838eaac0643de839e8fa77" 91 | }, 92 | "_ip": "127.0.0.1", 93 | "_or_port": 5001, 94 | "_dir_port": 7001, 95 | "_version": "Tor 0.3.0.7", 96 | "_flags": [ 97 | { 98 | "_enum": "RouterFlags.Authority" 99 | }, 100 | { 101 | "_enum": "RouterFlags.Exit" 102 | }, 103 | { 104 | "_enum": "RouterFlags.Fast" 105 | }, 106 | { 107 | "_enum": "RouterFlags.Guard" 108 | }, 109 | { 110 | "_enum": "RouterFlags.HSDir" 111 | }, 112 | { 113 | "_enum": "RouterFlags.Running" 114 | }, 115 | { 116 | "_enum": "RouterFlags.V2Dir" 117 | }, 118 | { 119 | "_enum": "RouterFlags.Valid" 120 | } 121 | ] 122 | }, 123 | { 124 | "_nickname": "test000a", 125 | "_fingerprint": { 126 | "_bytes": "de7242f8bbed366c7a930db7c75584f74a72223e" 127 | }, 128 | "_digest": { 129 | "_bytes": "1e0dcdc8fa8366845037c855239562e81fa9a1fc" 130 | }, 131 | "_ip": "127.0.0.1", 132 | "_or_port": 5000, 133 | "_dir_port": 7000, 134 | "_version": "Tor 0.3.0.7", 135 | "_flags": [ 136 | { 137 | "_enum": "RouterFlags.Authority" 138 | }, 139 | { 140 | "_enum": "RouterFlags.Exit" 141 | }, 142 | { 143 | "_enum": "RouterFlags.Fast" 144 | }, 145 | { 146 | "_enum": "RouterFlags.Guard" 147 | }, 148 | { 149 | "_enum": "RouterFlags.HSDir" 150 | }, 151 | { 152 | "_enum": "RouterFlags.Running" 153 | }, 154 | { 155 | "_enum": "RouterFlags.Stable" 156 | }, 157 | { 158 | "_enum": "RouterFlags.V2Dir" 159 | }, 160 | { 161 | "_enum": "RouterFlags.Valid" 162 | } 163 | ] 164 | } 165 | ], 166 | "signatures": [ 167 | { 168 | "algorithm": "sha1", 169 | "identity": "596CD48D61FDA4E868F4AA10FF559917BE3B1A35", 170 | "signing_key_digest": "9FBF54D6A62364320308A615BF4CF6B27B254FAD", 171 | "signature": { 172 | "_bytes": "1e8d2b2e88df2c7b3d7123c5c5eeb39ee1ae154f01bd1afa8271f58142d38d464ed0d490be8e4deb6f0a15e02cabea5312589e41e57a51fc2760dd54dada2684162fdfefe9d71071612e684c30084f12d462f1f8a4b633d3d56eb70165145b54affb1ab5a8ee38c493d0cdf0082042e11bb9a39152925c094cf27fb564570c2107bff379654524a6735d9feeed15267d785acfc4f710bc1d390444a8ec2e3ff999d954eff076e39b498bb4e1e5b04c4a20da09125a6b33b4aa17a71d501abb46b0d2f6d96b3dc921833bfca4145cb191cbb207903ca0fc9e751ec3ac951f09463207ca3e5cb370383902171966b453a3722df9e9fa2c80b88cd6c56a09490d74" 173 | } 174 | }, 175 | { 176 | "algorithm": "sha1", 177 | "identity": "BCB380A633592C218757BEE11E630511A485658A", 178 | "signing_key_digest": "9CA027E05B0CE1500D90DA13FFDA8EDDCD40A734", 179 | "signature": { 180 | "_bytes": "ba202df08af6ee96055f97cd2a25590e86ba10b544b60fc4dd87981c09e5492473a5a70b30c4cdfc7845fff66fbfc66314a4fde6fefbc4abc4e9bf2cec98c1ddea7a728896e2d90baa30e238d1ba88344a066cba0fe45981fd4d3319778166994a120d39124c5bd9e096705bcac3fb7a03e25f8f70d628156502ccd42da9a6984aeae3b910670aea70c91782d50c9ec116fb76ca163e82c886de84ab86c3f4b2742715fbc479414efebda045527516831752db5a382b668f6ebcc52d3c799d26312faeea01c4335758930a40f8bcc4f46e1a90f461bd18cc0da17444cb8b43ba58e6fe38b84e4e5f99ff18a5918335590c7fae6e5e81baaa50c584176876798b" 181 | } 182 | } 183 | ] 184 | } -------------------------------------------------------------------------------- /tests/unittest/test_consensus.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from torpy.consesus import DirectoryServer, DirectoryFlags, RouterFlags 4 | 5 | 6 | @pytest.mark.parametrize( 7 | 'line, result', 8 | [ 9 | ( 10 | '"moria1 orport=9101 v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31"', # noqa: E501, E126 11 | {'_nickname': 'moria1', '_fingerprint': b'\x96\x95\xdf\xc3_\xfe\xb8a2\x9b\x9f\x1a\xb0LF9p \xce1', 12 | '_digest': None, '_ip': '128.31.0.39', '_or_port': 9101, '_dir_port': 9131, '_version': None, 13 | '_flags': [RouterFlags.Authority], '_consensus': None, '_service_key': None, 14 | '_dir_flags': DirectoryServer.AUTH_FLAGS, '_v3ident': 'D586D18309DED4CD6D57C18FDB97EFA96D330566', 15 | '_ipv6': None, 16 | '_bridge': False}), 17 | ( 18 | '"tor26 orport=443 v3ident=14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 ipv6=[2001:858:2:2:aabb:0:563b:1526]:443 86.59.21.38:80 847B 1F85 0344 D787 6491 A548 92F9 0493 4E4E B85D"', # noqa: E501 19 | {'_nickname': 'tor26', '_fingerprint': b'\x84{\x1f\x85\x03D\xd7\x87d\x91\xa5H\x92\xf9\x04\x93NN\xb8]', 20 | '_digest': None, '_ip': '86.59.21.38', '_or_port': 443, '_dir_port': 80, '_version': None, 21 | '_flags': [RouterFlags.Authority], '_consensus': None, '_service_key': None, 22 | '_dir_flags': DirectoryServer.AUTH_FLAGS, '_v3ident': '14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4', 23 | '_ipv6': '[2001:858:2:2:aabb:0:563b:1526]:443', '_bridge': False}), 24 | ( 25 | '"dizum orport=443 v3ident=E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58 45.66.33.45:80 7EA6 EAD6 FD83 083C 538F 4403 8BBF A077 587D D755"', # noqa: E501 26 | {'_nickname': 'dizum', '_fingerprint': b'~\xa6\xea\xd6\xfd\x83\x08= self._next_len: 111 | send_buff = self._data[:self._next_len] 112 | self._data = self._data[self._next_len:] 113 | 114 | self._next_len = self._cells_builder.send(send_buff) 115 | if self._next_len is None: 116 | # New cell was built 117 | cell = next(self._cells_builder) 118 | yield cell 119 | self._next_len = next(self._cells_builder) 120 | logger.debug('Need more data (%i bytes, has %i bytes)', self._next_len, len(self._data)) 121 | 122 | def _cells_builder_gen(self): 123 | while self._socket: 124 | circuit_id, command_num = yield from self._read_by_format(self._protocol.header_format) 125 | cell_type = TorCommands.get_by_num(command_num) 126 | payload = yield from self._read_command_payload(cell_type) 127 | # logger.debug("recv from socket: circuit_id = %x, command = %s,\n" 128 | # "payload = %s", circuit_id, cell_type.__name__, to_hex(payload)) 129 | cell = self._protocol.deserialize(cell_type, payload, circuit_id) 130 | yield None 131 | yield cell 132 | 133 | def _read_command_payload(self, cell_type): 134 | if cell_type.is_var_len(): 135 | length, = yield from self._read_by_format(self._protocol.length_format) 136 | else: 137 | length = TorCell.MAX_PAYLOAD_SIZE 138 | cell_buff = yield from coro_recv_exact(length) 139 | return cell_buff 140 | 141 | def _read_by_format(self, struct_fmt): 142 | size = struct.calcsize(struct_fmt) 143 | data = yield from coro_recv_exact(size) 144 | if not data: 145 | raise NoDataException() 146 | return struct.unpack(struct_fmt, data) 147 | 148 | 149 | class NoDataException(Exception): 150 | pass 151 | 152 | 153 | class TorProtocol: 154 | DEFAULT_VERSION = 3 155 | SUPPORTED_VERSION = [3, 4] 156 | 157 | def __init__(self, version=DEFAULT_VERSION): 158 | self._version = version 159 | 160 | @property 161 | def version(self): 162 | return self._version 163 | 164 | @version.setter 165 | def version(self, version): 166 | self._version = version 167 | 168 | @property 169 | def header_format(self): 170 | # CircuitID [CIRCUIT_ID_LEN octets] 171 | # Command [1 byte] 172 | if self.version < 4: 173 | return '!HB' 174 | else: 175 | # Link protocol 4 increases circuit ID width to 4 bytes. 176 | return '!IB' 177 | 178 | @property 179 | def length_format(self): 180 | # Length [2 octets; big-endian integer] 181 | return '!H' 182 | 183 | def deserialize(self, command, payload, circuit_id=0): 184 | # parse depending on version 185 | # ... 186 | return TorCell.deserialize(command, circuit_id, payload, self.version) 187 | 188 | def serialize(self, cell): 189 | # get bytes depending on version 190 | # ... 191 | return cell.serialize(self.version) 192 | 193 | 194 | class TorHandshake: 195 | def __init__(self, tor_socket, tor_protocol): 196 | self.tor_socket = tor_socket 197 | self.tor_protocol = tor_protocol 198 | 199 | def initiate(self): 200 | # When the in-protocol handshake is used, the initiator sends a 201 | # VERSIONS cell to indicate that it will not be renegotiating. The 202 | # responder sends a VERSIONS cell, a CERTS cell (4.2 below) to give the 203 | # initiator the certificates it needs to learn the responder's 204 | # identity, an AUTH_CHALLENGE cell (4.3) that the initiator must include 205 | # as part of its answer if it chooses to authenticate, and a NET_INFO 206 | # cell (4.5). As soon as it gets the CERTS cell, the initiator knows 207 | # whether the responder is correctly authenticated. At this point the 208 | # initiator behaves differently depending on whether it wants to 209 | # authenticate or not. If it does not want to authenticate, it MUST 210 | # send a NET_INFO cell. 211 | self._send_versions() 212 | self.tor_protocol.version = self._retrieve_versions() 213 | 214 | self._retrieve_certs() 215 | 216 | self._retrieve_net_info() 217 | self._send_net_info() 218 | 219 | def _send_versions(self): 220 | """ 221 | Send CellVersion. 222 | 223 | When the "in-protocol" handshake is used, implementations MUST NOT 224 | list any version before 3, and SHOULD list at least version 3. 225 | 226 | Link protocols differences are: 227 | 1 -- The "certs up front" handshake. 228 | 2 -- Uses the renegotiation-based handshake. Introduces 229 | variable-length cells. 230 | 3 -- Uses the in-protocol handshake. 231 | 4 -- Increases circuit ID width to 4 bytes. 232 | 5 -- Adds support for link padding and negotiation (padding-spec.txt). 233 | """ 234 | self.tor_socket.send_cell(CellVersions(self.tor_protocol.SUPPORTED_VERSION)) 235 | 236 | def _retrieve_versions(self): 237 | cell = self.tor_socket.recv_cell() 238 | assert isinstance(cell, CellVersions) 239 | 240 | logger.debug('Remote protocol versions: %s', cell.versions) 241 | # Choose maximum supported by both 242 | return min(max(self.tor_protocol.SUPPORTED_VERSION), max(cell.versions)) 243 | 244 | def _retrieve_certs(self): 245 | logger.debug('Retrieving CERTS cell...') 246 | cell_certs = self.tor_socket.recv_cell() 247 | 248 | assert isinstance(cell_certs, CellCerts) 249 | # TODO: check certs validity 250 | 251 | logger.debug('Retrieving AUTH_CHALLENGE cell...') 252 | cell_auth = self.tor_socket.recv_cell() 253 | assert isinstance(cell_auth, CellAuthChallenge) 254 | 255 | def _retrieve_net_info(self): 256 | logger.debug('Retrieving NET_INFO cell...') 257 | cell = self.tor_socket.recv_cell() 258 | assert isinstance(cell, CellNetInfo) 259 | logger.debug('Our public IP address: %s', cell.this_or) 260 | 261 | def _send_net_info(self): 262 | """If version 2 or higher is negotiated, each party sends the other a NETINFO cell.""" 263 | logger.debug('Sending NET_INFO cell...') 264 | self.tor_socket.send_cell(CellNetInfo(int(time.time()), self.tor_socket.ip_address, '0')) 265 | -------------------------------------------------------------------------------- /torpy/cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torpyorg/torpy/ebf000cc93d2d7b3ddf68f2604afd46c998ae1c1/torpy/cli/__init__.py -------------------------------------------------------------------------------- /torpy/cli/console.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import logging 17 | import textwrap 18 | from argparse import ArgumentParser 19 | 20 | from torpy.utils import register_logger 21 | from torpy.http.urlopener import do_request as urllib_request 22 | try: 23 | from torpy.http.requests import do_request as requests_request 24 | except ImportError: 25 | requests_request = None 26 | 27 | logger = logging.getLogger(__name__) 28 | 29 | 30 | def print_data(data, to_file=None): 31 | if to_file: 32 | logger.info('Writing to file %s', to_file) 33 | with open(to_file, 'w+') as f: 34 | f.write(data) 35 | else: 36 | logger.warning(textwrap.indent(data, '> ', lambda line: True)) 37 | 38 | 39 | def main(): 40 | parser = ArgumentParser() 41 | parser.add_argument('--url', help='url', required=True) 42 | parser.add_argument('--method', default='GET', type=str.upper, help='http method') 43 | parser.add_argument('--data', default=None, help='http data') 44 | parser.add_argument('--hops', default=3, help='hops count', type=int) 45 | parser.add_argument('--to-file', default=None, help='save result to file') 46 | parser.add_argument('--header', default=None, dest='headers', nargs=2, action='append', help='set some http header') 47 | parser.add_argument('--auth-data', nargs=2, action='append', help='set auth data for hidden service authorization') 48 | parser.add_argument('--log-file', default=None, help='log file path') 49 | parser.add_argument('--requests-lib', dest='request_func', default=urllib_request, action='store_const', 50 | const=requests_request, help='use requests library for making requests') 51 | parser.add_argument('-v', '--verbose', default=0, help='enable verbose output', action='count') 52 | args = parser.parse_args() 53 | 54 | register_logger(args.verbose, log_file=args.log_file) 55 | 56 | if not args.request_func: 57 | raise Exception('Requests library not installed, use default urllib') 58 | 59 | data = args.request_func(args.url, method=args.method, data=args.data, headers=args.headers, hops=args.hops, 60 | auth_data=args.auth_data, verbose=args.verbose) 61 | print_data(data, args.to_file) 62 | 63 | 64 | if __name__ == '__main__': 65 | try: 66 | main() 67 | except KeyboardInterrupt: 68 | logger.error('Interrupted.') 69 | -------------------------------------------------------------------------------- /torpy/cli/socks.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | """Socks5 repeater proxy.""" 17 | import os 18 | import array 19 | import select 20 | import socket 21 | import struct 22 | import logging 23 | import threading 24 | from argparse import ArgumentParser 25 | from contextlib import contextmanager 26 | 27 | from torpy.utils import recv_exact, register_logger 28 | from torpy.client import TorClient 29 | 30 | logger = logging.getLogger(__name__) 31 | 32 | 33 | class SocksProxy: 34 | def __init__(self, server_sock, client_sock): 35 | self.server_sock = server_sock 36 | self.client_sock = client_sock 37 | 38 | def run(self): 39 | ssock = self.server_sock 40 | csock = self.client_sock 41 | addr, port = csock.getsockname() 42 | csock.sendall(b'\x05\0\0\x01\x7f\0\0\x01' + struct.pack('!H', port)) 43 | try: 44 | while True: 45 | r, w, _ = select.select([ssock, csock], [], []) 46 | if ssock in r: 47 | buf = ssock.recv(4096) 48 | if len(buf) == 0: 49 | break 50 | csock.send(buf) 51 | if csock in r: 52 | buf = csock.recv(4096) 53 | if len(buf) == 0: 54 | break 55 | ssock.send(buf) 56 | except BaseException: 57 | logger.exception('[socks] Some error') 58 | finally: 59 | logger.info('[socks] Close ssock') 60 | ssock.close() 61 | logger.info('[socks] Close csock') 62 | csock.close() 63 | 64 | 65 | class SocksServer(object): 66 | def __init__(self, circuit, ip, port): 67 | self.circuit = circuit 68 | self.ip = ip 69 | self.port = port 70 | self.listen_socket = None 71 | 72 | def __enter__(self): 73 | """Start listen incoming connections.""" 74 | lsock = self.listen_socket = socket.socket(2, 1, 6) 75 | lsock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 76 | lsock.bind((self.ip, self.port)) 77 | logger.info('Start socks proxy at %s:%s', self.ip, self.port) 78 | lsock.listen(0) 79 | return self 80 | 81 | def __exit__(self, exc_type, exc_val, exc_tb): 82 | """Close listen incoming connections.""" 83 | self.listen_socket.close() 84 | if exc_type: 85 | from traceback import format_exception 86 | 87 | logger.error( 88 | '[socks] Exception in server:\n%s', 89 | '\n'.join(format_exception(exc_type, exc_val, exc_tb)).rstrip('\r\n'), 90 | ) 91 | 92 | def start(self): 93 | while True: 94 | try: 95 | csock, caddr = self.listen_socket.accept() 96 | except BaseException: 97 | logger.info('[socks] Closing by user request') 98 | raise 99 | logger.info('[socks] Client connected %s', caddr) 100 | Socks5(self.circuit, csock, caddr).start() 101 | 102 | 103 | class Socks5(threading.Thread): 104 | def __init__(self, circuit, client_sock, client_addr): 105 | if client_addr[0] == '127.0.0.1': 106 | thread_name = 'Socks-%s' % client_addr[1] 107 | else: 108 | thread_name = 'Socks-%s:%s' % (client_addr[0], client_addr[1]) 109 | super().__init__(name=thread_name) 110 | self.circuit = circuit 111 | self.client_sock = client_sock 112 | self.client_addr = client_addr 113 | 114 | def error(self, err=b'\x01\0'): 115 | try: 116 | self.client_sock.send(b'\x05' + err) 117 | self.client_sock.close() 118 | self.client_sock = None 119 | except BaseException: 120 | pass 121 | 122 | @contextmanager 123 | def create_socket(self, dst, port): 124 | logger.info('[socks] Connecting to %s:%s', dst, port) 125 | with self.circuit.create_stream((dst, port)) as tor_stream: 126 | yield tor_stream.create_socket() 127 | logger.debug('[socks] Closing stream #%x', tor_stream.id) 128 | 129 | def run(self): 130 | csock = self.client_sock 131 | try: 132 | ver = csock.recv(1) 133 | if ver != b'\x05': 134 | return self.error(b'\xff') 135 | nmeth, = array.array('B', csock.recv(1)) 136 | _ = recv_exact(csock, nmeth) # read methods 137 | csock.send(b'\x05\0') 138 | hbuf = recv_exact(csock, 4) 139 | if not hbuf: 140 | return self.error() 141 | 142 | ver, cmd, rsv, atyp = list(hbuf) 143 | if ver != 5 and cmd != 1: 144 | return self.error() 145 | 146 | if atyp == 1: 147 | dst = '.'.join(str(i) for i in recv_exact(csock, 4)) 148 | elif atyp == 3: 149 | n, = array.array('B', csock.recv(1)) 150 | dst = recv_exact(csock, n).decode() 151 | elif atyp == 4: 152 | dst = ':'.join(recv_exact(csock, 2).hex() for _ in range(8)) 153 | # TODO: ipv6 154 | return self.error() 155 | else: 156 | return self.error() 157 | 158 | port = int(recv_exact(csock, 2).hex(), 16) 159 | 160 | with self.create_socket(dst, port) as ssock: 161 | SocksProxy(ssock, csock).run() 162 | except Exception: 163 | logger.exception('[socks] csock close by exception') 164 | csock.close() 165 | self.client_sock = None 166 | 167 | 168 | def main(): 169 | parser = ArgumentParser(description=__doc__, prog=os.path.basename(__file__)) 170 | parser.add_argument('-i', '--ip', default='127.0.0.1', help='ip address to bind to') 171 | parser.add_argument('-p', '--port', default=1050, type=int, help='bind port') 172 | parser.add_argument('--hops', default=3, help='hops count', type=int) 173 | parser.add_argument('-v', '--verbose', help='enable verbose output', action='store_true') 174 | args = parser.parse_args() 175 | 176 | register_logger(args.verbose) 177 | 178 | tor = TorClient() 179 | with tor.create_circuit(args.hops) as circuit, SocksServer(circuit, args.ip, args.port) as socks_serv: 180 | socks_serv.start() 181 | 182 | 183 | if __name__ == '__main__': 184 | main() 185 | -------------------------------------------------------------------------------- /torpy/client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import socket 17 | import logging 18 | import functools 19 | from typing import TYPE_CHECKING 20 | from contextlib import contextmanager 21 | 22 | from torpy.guard import TorGuard 23 | from torpy.utils import retry, log_retry 24 | from torpy.circuit import TorCircuit 25 | from torpy.cell_socket import TorSocketConnectError 26 | from torpy.consesus import TorConsensus 27 | from torpy.cache_storage import TorCacheDirStorage 28 | 29 | if TYPE_CHECKING: 30 | from typing import ContextManager 31 | 32 | logger = logging.getLogger(__name__) 33 | 34 | 35 | class TorClient: 36 | def __init__(self, consensus=None, auth_data=None): 37 | self._consensus = consensus or TorConsensus() 38 | self._auth_data = auth_data or {} 39 | 40 | @classmethod 41 | def create(cls, authorities=None, cache_class=None, cache_kwargs=None, auth_data=None): 42 | cache_class = cache_class or TorCacheDirStorage 43 | cache_kwargs = cache_kwargs or {} 44 | consensus = TorConsensus(authorities=authorities, cache_storage=cache_class(**cache_kwargs)) 45 | return cls(consensus, auth_data) 46 | 47 | @retry(3, BaseException, log_func=functools.partial(log_retry, 48 | msg='Retry with another guard...', 49 | no_traceback=(socket.timeout, TorSocketConnectError,)) 50 | ) 51 | def get_guard(self, by_flags=None): 52 | # TODO: add another stuff to filter guards 53 | guard_router = self._consensus.get_random_guard_node(by_flags) 54 | return TorGuard(guard_router, purpose='TorClient', consensus=self._consensus, auth_data=self._auth_data) 55 | 56 | @contextmanager 57 | def create_circuit(self, hops_count=3, guard_by_flags=None) -> 'ContextManager[TorCircuit]': 58 | with self.get_guard(guard_by_flags) as guard: 59 | yield guard.create_circuit(hops_count) 60 | 61 | def __enter__(self): 62 | """Start using the tor client.""" 63 | return self 64 | 65 | def __exit__(self, exc_type, exc_val, exc_tb): 66 | """Close the tor client.""" 67 | self.close() 68 | 69 | def close(self): 70 | self._consensus.close() 71 | -------------------------------------------------------------------------------- /torpy/crypto.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import os 17 | import logging 18 | 19 | from torpy.crypto_common import sha1, aes_update, rsa_encrypt, rsa_load_der, aes_ctr_encryptor 20 | 21 | logger = logging.getLogger(__name__) 22 | 23 | 24 | TOR_DIGEST_LEN = 20 25 | 26 | 27 | def tor_digest(msg): 28 | return sha1(msg) 29 | 30 | 31 | def kdf_tor(shared_secret): 32 | # tor ref: crypto_expand_key_material_TAP 33 | t = shared_secret + bytes([0]) 34 | computed_auth = tor_digest(t) 35 | key_material = b'' 36 | for i in range(1, 5): 37 | t = shared_secret + bytes([i]) 38 | tsh = tor_digest(t) 39 | key_material += tsh 40 | return computed_auth, key_material 41 | 42 | 43 | # tor-spec.txt 0.3. 44 | KEY_LEN = 16 45 | PK_ENC_LEN = 128 46 | PK_PAD_LEN = 42 47 | 48 | PK_DATA_LEN = PK_ENC_LEN - PK_PAD_LEN 49 | PK_DATA_LEN_WITH_KEY = PK_DATA_LEN - KEY_LEN 50 | 51 | 52 | def hybrid_encrypt(data, rsa_key_der): 53 | """ 54 | Hybrid encryption scheme. 55 | 56 | Encrypt the entire contents of the byte array "data" with the given "TorPublicKey" according to 57 | the "hybrid encryption" scheme described in the main Tor specification (tor-spec.txt). 58 | """ 59 | rsa_key = rsa_load_der(rsa_key_der) 60 | 61 | if len(data) < PK_DATA_LEN: 62 | return rsa_encrypt(rsa_key, data) 63 | 64 | aes_key_bytes = os.urandom(KEY_LEN) 65 | 66 | # RSA(K | M1) --> C1 67 | m1 = data[:PK_DATA_LEN_WITH_KEY] 68 | c1 = rsa_encrypt(rsa_key, aes_key_bytes + m1) 69 | 70 | # AES_CTR(M2) --> C2 71 | m2 = data[PK_DATA_LEN_WITH_KEY:] 72 | aes_key = aes_ctr_encryptor(aes_key_bytes) 73 | c2 = aes_update(aes_key, m2) 74 | 75 | return c1 + c2 76 | -------------------------------------------------------------------------------- /torpy/crypto_common.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import base64 17 | import hashlib 18 | from hmac import compare_digest 19 | 20 | from cryptography.hazmat.backends import default_backend 21 | from cryptography.hazmat.primitives import hashes, serialization 22 | from cryptography.hazmat.primitives.hmac import HMAC 23 | from cryptography.hazmat.primitives.ciphers import Cipher 24 | from cryptography.hazmat.primitives.kdf.hkdf import HKDFExpand 25 | from cryptography.hazmat.primitives.asymmetric import dh, padding 26 | from cryptography.hazmat.primitives.ciphers.modes import CTR 27 | from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PublicKey, X25519PrivateKey 28 | from cryptography.hazmat.primitives.ciphers.algorithms import AES 29 | 30 | bend = default_backend() 31 | 32 | 33 | def b64decode(data): 34 | return base64.b64decode(data + '=' * (len(data) % 4)) 35 | 36 | 37 | def sha1(msg): 38 | sha = hashes.Hash(hashes.SHA1(), backend=bend) 39 | sha.update(msg) 40 | return sha.finalize() 41 | 42 | 43 | def sha3_256(msg): 44 | sha = hashes.Hash(hashes.SHA3_256(), backend=bend) 45 | sha.update(msg) 46 | return sha.finalize() 47 | 48 | 49 | def hash_stream(name): 50 | return hashlib.new(name) 51 | 52 | 53 | def hash_update(hash, msg): 54 | return hash.update(msg) 55 | 56 | 57 | def hash_finalize(hash): 58 | return hash.digest() 59 | 60 | 61 | def sha1_stream(init_msg=None): 62 | hash = hashes.Hash(hashes.SHA1(), backend=bend) 63 | if init_msg: 64 | sha1_stream_update(hash, init_msg) 65 | return hash 66 | 67 | 68 | def sha1_stream_update(hash, msg): 69 | hash.update(msg) 70 | return hash 71 | 72 | 73 | def sha1_stream_clone(hash): 74 | return hash.copy() 75 | 76 | 77 | def sha1_stream_finalize(hash): 78 | return hash.finalize() 79 | 80 | 81 | def hmac(key, msg): 82 | hmac = HMAC(key, algorithm=hashes.SHA256(), backend=bend) 83 | hmac.update(msg) 84 | return hmac.finalize() 85 | 86 | 87 | def hkdf_sha256(key, length=16, info=''): 88 | hkdf = HKDFExpand(algorithm=hashes.SHA256(), length=length, info=info, backend=bend) 89 | return hkdf.derive(key) 90 | 91 | 92 | def curve25519_private(): 93 | return X25519PrivateKey.generate() 94 | 95 | 96 | def curve25519_get_shared(private, public): 97 | return private.exchange(public) 98 | 99 | 100 | def curve25519_public_from_private(private): 101 | return private.public_key() 102 | 103 | 104 | def curve25519_public_from_bytes(data): 105 | return X25519PublicKey.from_public_bytes(data) 106 | 107 | 108 | def curve25519_to_bytes(key): 109 | if isinstance(key, X25519PublicKey): 110 | return key.public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) 111 | else: 112 | return key.private_bytes( 113 | serialization.Encoding.Raw, serialization.PrivateFormat.Raw, serialization.NoEncryption() 114 | ) 115 | 116 | 117 | _P = 179769313486231590770839156793787453197860296048756011706444423684197180216158519368947833795864925541502180565485980503646440548199239100050792877003355816639229553136239076508735759914822574862575007425302077447712589550957937778424442426617334727629299387668709205606050270810842907692932019128194467627007 # noqa: E501 118 | _G = 2 119 | _DH_PARAMETERS_NUMBERS = dh.DHParameterNumbers(_P, _G) 120 | _DH_PARAMETERS = _DH_PARAMETERS_NUMBERS.parameters(default_backend()) 121 | 122 | 123 | def dh_private(): 124 | return _DH_PARAMETERS.generate_private_key() 125 | 126 | 127 | def dh_public(private): 128 | return private.public_key() 129 | 130 | 131 | def dh_public_to_bytes(key): 132 | return key.public_numbers().y.to_bytes(128, 'big') 133 | 134 | 135 | def dh_public_from_bytes(public_bytes): 136 | y = int.from_bytes(public_bytes, byteorder='big') 137 | peer_public_numbers = dh.DHPublicNumbers(y, _DH_PARAMETERS.parameter_numbers()) 138 | return peer_public_numbers.public_key(default_backend()) 139 | 140 | 141 | def dh_shared(private_key, another_public): 142 | return private_key.exchange(another_public) 143 | 144 | 145 | def rsa_load_der(public_der_data): 146 | return serialization.load_der_public_key(public_der_data, backend=bend) 147 | 148 | 149 | def rsa_load_pem(public_pem_data): 150 | return serialization.load_pem_public_key(public_pem_data, backend=bend) 151 | 152 | 153 | def rsa_verify(pubkey, sig, dig): 154 | dig_size = len(dig) 155 | sig_int = int(sig.hex(), 16) 156 | pn = pubkey.public_numbers() 157 | decoded = pow(sig_int, pn.e, pn.n) 158 | buf = '%x' % decoded 159 | if len(buf) % 2: 160 | buf = '0' + buf 161 | buf = '00' + buf 162 | hash_buf = bytes.fromhex(buf) 163 | 164 | pad_type = b'\0\1' 165 | pad_len = len(hash_buf) - 2 - 1 - dig_size 166 | cmp_dig = pad_type + b'\xff' * pad_len + b'\0' + dig 167 | return compare_digest(hash_buf, cmp_dig) 168 | 169 | 170 | def rsa_encrypt(key, data): 171 | return key.encrypt( 172 | data, padding.OAEP(mgf=padding.MGF1(algorithm=hashes.SHA1()), algorithm=hashes.SHA1(), label=None) 173 | ) 174 | 175 | 176 | def aes_ctr_encryptor(key, iv=b'\0' * 16): 177 | return Cipher(AES(key), CTR(iv), backend=bend).encryptor() 178 | 179 | 180 | def aes_ctr_decryptor(key, iv=b'\0' * 16): 181 | return Cipher(AES(key), CTR(iv), backend=bend).decryptor() 182 | 183 | 184 | def aes_update(aes_cipher, data): 185 | return aes_cipher.update(data) 186 | -------------------------------------------------------------------------------- /torpy/crypto_state.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import struct 17 | import logging 18 | 19 | from torpy.cells import RelayedTorCell 20 | from torpy.utils import to_hex 21 | from torpy.crypto_common import ( 22 | aes_update, 23 | sha1_stream, 24 | aes_ctr_decryptor, 25 | aes_ctr_encryptor, 26 | sha1_stream_clone, 27 | sha1_stream_update, 28 | sha1_stream_finalize, 29 | ) 30 | 31 | logger = logging.getLogger(__name__) 32 | 33 | 34 | class CryptoState: 35 | def __init__(self, data): 36 | """ 37 | Parse handshake data and create forward/backward digests. 38 | 39 | When used in the ntor handshake, the first HASH_LEN bytes form the 40 | forward digest Df; the next HASH_LEN form the backward digest Db; the 41 | next KEY_LEN form Kf, the next KEY_LEN form Kb, and the final 42 | DIGEST_LEN bytes are taken as a nonce to use in the place of KH in the 43 | hidden service protocol. Excess bytes from K are discarded. 44 | 45 | :type data: bytes 46 | """ 47 | (_fdig, _bdig, _ekey, _dkey) = struct.unpack('!20s20s16s16s', data) 48 | 49 | self._forward_digest = sha1_stream(_fdig) 50 | self._backward_digest = sha1_stream(_bdig) 51 | 52 | self._forward_cipher = aes_ctr_encryptor(_ekey) 53 | self._backward_cipher = aes_ctr_decryptor(_dkey) 54 | 55 | def _digesting_func(self, payload): 56 | self._forward_digest.update(payload) 57 | digest = self._forward_digest.copy() 58 | return digest.finalize()[:4] 59 | 60 | def _encrypting_func(self, payload): 61 | return aes_update(self._forward_cipher, payload) 62 | 63 | def _digest_check_func(self, payload, digest): 64 | digest_clone = sha1_stream_clone(self._backward_digest) 65 | sha1_stream_update(digest_clone, payload) 66 | new_digest = sha1_stream_finalize(digest_clone)[:4] 67 | if new_digest != digest: 68 | logger.debug( 69 | 'received cell digest not equal ({!r} != {!r}); payload = {!r}'.format( 70 | to_hex(new_digest), to_hex(digest), to_hex(payload) 71 | ) 72 | ) 73 | return False 74 | 75 | sha1_stream_update(self._backward_digest, payload) 76 | return True 77 | 78 | def _decrypting_func(self, payload): 79 | return aes_update(self._backward_cipher, payload) 80 | 81 | def encrypt_forward(self, relay_cell): 82 | if not relay_cell.digest: 83 | relay_cell.prepare(self._digesting_func) 84 | relay_cell.encrypt(self._encrypting_func) 85 | 86 | def decrypt_backward(self, relay_cell): 87 | # tor ref: relay_decrypt_cell 88 | encrypted = relay_cell.get_encrypted() 89 | payload = self._decrypting_func(encrypted) 90 | 91 | # Check if cell is recognized 92 | header = RelayedTorCell.parse_header(payload) 93 | if header['is_recognized'] == 0: 94 | payload_copy = RelayedTorCell.set_header_digest(payload, b'\0' * 4) 95 | 96 | # tor ref: relay_digest_matches 97 | if self._digest_check_func(payload_copy, header['digest']): 98 | relay_cell.set_decrypted(**header) 99 | return 100 | # Treat as encrypted even if is_recognized flag is zero but digests are not equal: 101 | # collisions are possible in the encrypted buffer 102 | 103 | # Still encrypted 104 | relay_cell.set_encrypted(payload) 105 | -------------------------------------------------------------------------------- /torpy/documents/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | from torpy.documents.basics import TorDocument 17 | from torpy.documents.factory import TorDocumentsFactory 18 | 19 | 20 | __all__ = ['TorDocument', 'TorDocumentsFactory'] 21 | -------------------------------------------------------------------------------- /torpy/documents/basics.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import io 17 | 18 | from torpy.crypto_common import hash_stream, hash_update, hash_finalize 19 | from torpy.documents.items import ItemObject, ItemMask 20 | 21 | 22 | class TorDocumentObject: 23 | START_ITEM = None 24 | ITEMS = None 25 | CLASS = None 26 | 27 | def __init__(self, check_start=False): 28 | if self.START_ITEM is None or self.ITEMS is None: 29 | raise Exception('You must fill items for this object') 30 | self._check_start = check_start 31 | self._fields = {} 32 | 33 | def __getattr__(self, item): 34 | """Hooks fields search.""" 35 | if item in self._fields: 36 | return self._fields[item] 37 | 38 | @classmethod 39 | def from_item_result(cls, item, result): 40 | o = cls() 41 | o._update(item, result) 42 | return o 43 | 44 | def _update(self, item, result): 45 | if item.as_list: 46 | key = item.out_name 47 | if key not in self._fields: 48 | self._fields[key] = [] 49 | self._fields[key].append(result) 50 | else: 51 | if type(result) is dict: 52 | self._fields.update(result) 53 | else: 54 | self._fields[item.out_name] = result 55 | 56 | 57 | class TorDocumentReader: 58 | def __init__(self, document_string, digests_names=None, digest_start=None, digest_end=None): 59 | self._f = io.StringIO(document_string) 60 | self._digests_names = digests_names or [] 61 | self._digests = [hash_stream(digest) for digest in self._digests_names] 62 | self._digest_start = digest_start 63 | self._digest_end = digest_end 64 | self._digesting = False 65 | 66 | def _update_digests(self, data): 67 | for h in self._digests: 68 | hash_update(h, data.encode('utf-8')) 69 | 70 | def get_digests(self): 71 | return {self._digests_names[i]: hash_finalize(h) for i, h in enumerate(self._digests)} 72 | 73 | def lines_gen(self): 74 | for line in self._f: 75 | if self._digests_names: 76 | if not self._digesting: 77 | i = line.find(self._digest_start) 78 | if i >= 0: 79 | self._update_digests(line[i:]) 80 | self._digesting = True 81 | else: 82 | i = line.find(self._digest_end) 83 | if i >= 0: 84 | self._update_digests(line[:i + len(self._digest_end)]) 85 | self._digesting = False 86 | else: 87 | self._update_digests(line) 88 | 89 | yield line.rstrip('\n') 90 | 91 | 92 | class TorDocument(TorDocumentObject): 93 | DOCUMENT_NAME = None 94 | 95 | def __init__(self, raw_string, digests_names=None, digest_start=None, digest_end=None): 96 | if self.DOCUMENT_NAME is None: 97 | raise Exception('You must fill document name field') 98 | super().__init__(check_start=True) 99 | self._raw_string = raw_string 100 | self._digests = {} 101 | self._items, self._items_obj, self._items_mask = self._collect_items() 102 | self._read(digests_names, digest_start, digest_end) 103 | 104 | @classmethod 105 | def check_start(cls, raw_string): 106 | return raw_string.startswith(cls.START_ITEM.keyword + ' ') 107 | 108 | def get_digest(self, hash_name): 109 | return self._digests.get(hash_name, None) 110 | 111 | def _collect_items(self): 112 | items = {} 113 | items_obj = {} 114 | items_mask = [] 115 | 116 | if self.START_ITEM: 117 | items[self.START_ITEM.keyword] = self.START_ITEM, True 118 | 119 | for item in self.ITEMS: 120 | if type(item) is ItemObject: 121 | items_obj[item.object_cls.START_ITEM.keyword] = item.object_cls.START_ITEM, item, True 122 | for sub_item in item.object_cls.ITEMS: 123 | items_obj[sub_item.keyword] = sub_item, item, False 124 | elif isinstance(item, ItemMask): 125 | items_mask.append(item) 126 | else: 127 | items[item.keyword] = item, False 128 | 129 | return items, items_obj, items_mask 130 | 131 | def check_items(self, line, lines): 132 | t = line.split(' ', 1) 133 | if len(t) < 2: 134 | kw, rest = t[0], '' 135 | else: 136 | kw, rest = t 137 | 138 | if self._items: 139 | t = self._items.get(kw, None) 140 | if t: 141 | item, st = t 142 | if self._check_start: 143 | if not st: 144 | raise Exception( 145 | f'"{self.DOCUMENT_NAME}" document must start with "{self.START_ITEM.keyword} " item' 146 | ) 147 | else: 148 | self._check_start = False 149 | 150 | # Parse line 151 | result = item.parse_func(rest, lines, *item.parse_args) 152 | self._update(item, result) 153 | return 154 | 155 | if self._items_obj: 156 | t = self._items_obj.get(kw, None) 157 | if t: 158 | item, oitem, st = t 159 | if st: 160 | result = item.parse_func(rest, lines, *item.parse_args) 161 | if result: 162 | # Start new object 163 | obj = oitem.object_cls.from_item_result(item, result) 164 | self._update(oitem, obj) 165 | return 166 | else: 167 | obj_lst = getattr(self, oitem.out_name, None) 168 | if not obj_lst: 169 | # Skip this item because it's not started yet 170 | return 171 | result = item.parse_func(rest, lines, *item.parse_args) 172 | # Grab last one 173 | obj = obj_lst[-1] 174 | obj._update(item, result) 175 | return 176 | 177 | for item in self._items_mask: 178 | m = item.check_item(kw) 179 | if m: 180 | result = item.parse_func(line, m, lines) 181 | self._update(item, result) 182 | return 183 | return 184 | 185 | def _fix_objects(self): 186 | for t in self._items_obj.values(): 187 | if t: 188 | item, oitem, st = t 189 | if not oitem.object_cls.CLASS: 190 | continue 191 | obj_lst = getattr(self, oitem.out_name, None) 192 | for i, obj in enumerate(obj_lst): 193 | if not hasattr(obj, 'CLASS'): 194 | break 195 | obj_lst[i] = obj.CLASS(**obj._fields) 196 | 197 | def _read(self, digests_names, digest_start, digest_end): 198 | reader = TorDocumentReader(self._raw_string, digests_names, digest_start, digest_end) 199 | lines = reader.lines_gen() 200 | for line in lines: 201 | self.check_items(line, lines) 202 | self._fix_objects() 203 | self._digests = reader.get_digests() 204 | 205 | @property 206 | def raw_string(self): 207 | return self._raw_string 208 | -------------------------------------------------------------------------------- /torpy/documents/dir_key_certificate.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import logging 17 | 18 | from torpy.documents.basics import TorDocument, TorDocumentObject 19 | from torpy.documents.items import Item, ItemDate, ItemInt, ItemMulti, ItemObject 20 | 21 | logger = logging.getLogger(__name__) 22 | 23 | 24 | class DirKeyCertificateObject(TorDocumentObject): 25 | 26 | START_ITEM = ItemInt('dir-key-certificate-version') 27 | 28 | ITEMS = [ 29 | # fingerprint 14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 30 | Item('fingerprint'), 31 | # dir-key-published 2019-06-01 00:00:00 32 | ItemDate('dir-key-published'), 33 | # dir-key-expires 2019-11-01 00:00:00 34 | ItemDate('dir-key-expires'), 35 | ItemMulti('dir-identity-key', 'rsa public key'), 36 | ItemMulti('dir-signing-key', 'rsa public key'), 37 | ItemMulti('dir-key-crosscert', 'id signature'), 38 | ItemMulti('dir-key-certification', 'signature'), 39 | ] 40 | 41 | 42 | class DirKeyCertificate(TorDocument, DirKeyCertificateObject): 43 | DOCUMENT_NAME = 'dir_key_certificate' 44 | 45 | 46 | class DirKeyCertificateList(TorDocument): 47 | DOCUMENT_NAME = 'dir_key_certificates' 48 | 49 | START_ITEM = '' 50 | 51 | ITEMS = [ItemObject(DirKeyCertificateObject, out_name='certs')] 52 | 53 | def find(self, identity): 54 | return next((cert for cert in self.certs if cert.fingerprint == identity), None) 55 | -------------------------------------------------------------------------------- /torpy/documents/factory.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | from torpy.documents import TorDocument 17 | 18 | 19 | class TorDocumentsFactory: 20 | @staticmethod 21 | def parse(raw_string, kwargs=None, possible=None): 22 | kwargs = kwargs or {} 23 | possible = possible or TorDocument.__subclasses__() 24 | 25 | for doc_cls in possible: 26 | if doc_cls.check_start(raw_string): 27 | return doc_cls(raw_string, **kwargs) 28 | 29 | return None 30 | -------------------------------------------------------------------------------- /torpy/documents/items.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import re 17 | from datetime import datetime 18 | 19 | from torpy.crypto_common import b64decode 20 | 21 | 22 | class ItemType: 23 | # "Exactly once": These items MUST occur exactly one time in every 24 | # instance of the document type. 25 | ExactlyOnce = 1 26 | 27 | # "At most once": These items MAY occur zero or one times in any 28 | # instance of the document type, but MUST NOT occur more than once. 29 | AtMostOnce = 2 30 | 31 | # "Any number": These items MAY occur zero, one, or more times in any 32 | # instance of the document type. 33 | AnyNumber = 3 34 | 35 | # "Once or more": These items MUST occur at least once in any instance 36 | # of the document type, and MAY occur more. 37 | OnceOrMore = 4 38 | 39 | 40 | class ItemParsers: 41 | @staticmethod 42 | def split_symbol(line, _reader, split_symbol, fields_names, *_): 43 | split_line = line.split(split_symbol) 44 | fields = fields_names 45 | return dict(zip(fields, split_line)) 46 | 47 | @staticmethod 48 | def by_regex(line, *_): 49 | return line 50 | 51 | @staticmethod 52 | def store_string(line, *_): 53 | return line 54 | 55 | 56 | class Item: 57 | def __init__( 58 | self, 59 | keyword, 60 | parse_func=ItemParsers.store_string, 61 | parse_args=None, 62 | out_name=None, 63 | type=ItemType.ExactlyOnce, 64 | as_list=False, 65 | ): 66 | self.keyword = keyword 67 | self.parse_func = parse_func 68 | self.parse_args = parse_args or [] 69 | self.out_name = out_name or keyword.replace('-', '_') 70 | self.type = type 71 | self.as_list = as_list 72 | 73 | 74 | class ItemMask(Item): 75 | def check_item(self, kw): 76 | return self._mask.match(kw) 77 | 78 | def __init__(self, mask, parse_func, out_name, as_list=False): 79 | super().__init__('', parse_func=parse_func, out_name=out_name, as_list=as_list) 80 | self._mask = re.compile(mask) 81 | 82 | 83 | class ItemObject(Item): 84 | def __init__(self, object_cls, out_name): 85 | super().__init__('', parse_func=None, out_name=out_name, as_list=True) 86 | self.object_cls = object_cls 87 | 88 | 89 | class ItemDate(Item): 90 | @staticmethod 91 | def _get_date(line, *_): 92 | # Get only two spaced parts 93 | line = line.split(' ')[:2] 94 | # 2019-01-01 00:00:00 95 | return datetime.strptime(' '.join(line), '%Y-%m-%d %H:%M:%S') 96 | 97 | def __init__(self, keyword, out_name=None, type=ItemType.ExactlyOnce): 98 | super().__init__(keyword, parse_func=ItemDate._get_date, out_name=out_name, type=type) 99 | 100 | 101 | class ItemInt(Item): 102 | @staticmethod 103 | def _get_int(line, *_): 104 | # Get only one spaced part 105 | line = line.split(' ')[0] 106 | return int(line) 107 | 108 | def __init__(self, keyword, out_name=None, type=ItemType.ExactlyOnce): 109 | super().__init__(keyword, parse_func=ItemInt._get_int, out_name=out_name, type=type) 110 | 111 | 112 | class ItemEnum(Item): 113 | def _parse_enum(self, line, *_): 114 | flags = filter(lambda i: i in self._enum_values, line.split(' ')) 115 | flags = map(lambda i: self._enum_cls[i], flags) 116 | # reduce(ior, flags) 117 | return list(flags) 118 | 119 | def __init__(self, keyword, enum_cls, out_name=None, type=ItemType.ExactlyOnce): 120 | super().__init__(keyword, parse_func=self._parse_enum, out_name=out_name, type=type) 121 | self._enum_cls = enum_cls 122 | self._enum_values = dir(self._enum_cls) 123 | 124 | 125 | class ItemMulti(Item): 126 | def __init__(self, keyword, ml_name, parse_func=ItemParsers.store_string, out_name=None, as_list=False): 127 | super().__init__(keyword, parse_func=self._parse, out_name=out_name, as_list=as_list) 128 | self._args_parse_func = parse_func 129 | self._ml_name = ml_name.replace(' ', '_') 130 | self._ml_start_line = f'-----BEGIN {ml_name.upper()}-----' 131 | self._ml_end_line = f'-----END {ml_name.upper()}-----' 132 | 133 | def _parse(self, line, lines, *_): 134 | args = self._args_parse_func(line) 135 | ml = self._read_ml(lines) 136 | if args: 137 | args.update({self._ml_name: ml}) 138 | else: 139 | args = {self.out_name: ml} 140 | return args 141 | 142 | def _read_ml(self, lines): 143 | line = next(lines) 144 | if line != self._ml_start_line: 145 | raise Exception(f'Begin line for {self.keyword} not found') 146 | ml = '' 147 | for line in lines: 148 | if line == self._ml_end_line: 149 | break 150 | ml += line 151 | return b64decode(ml) 152 | -------------------------------------------------------------------------------- /torpy/documents/network_status_diff.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | from enum import Enum 17 | 18 | from torpy.documents import TorDocument 19 | from torpy.documents.items import ItemMask, ItemInt, Item, ItemParsers 20 | 21 | 22 | class EdActionType(Enum): 23 | Delete = 'd' 24 | Append = 'a' 25 | Change = 'c' 26 | 27 | 28 | class EdAction: 29 | def __init__(self, start, end, act_type, data): 30 | self.start = start 31 | self.end = end 32 | self.type = act_type 33 | self.data = data 34 | 35 | def apply(self, lines, cur_line, cur_end): 36 | start, end = self.start, self.end 37 | if start == '$': 38 | start = end = cur_line 39 | if end == '$': 40 | end = cur_end 41 | 42 | if self.type == EdActionType.Append: 43 | lines[start:start] = self.data 44 | return start + len(self.data) 45 | 46 | if self.type == EdActionType.Change: 47 | lines[start - 1:end] = self.data 48 | return start - 1 + len(self.data) 49 | 50 | if self.type == EdActionType.Delete: 51 | del lines[start - 1:end] 52 | return start 53 | 54 | 55 | class ItemAction(ItemMask): 56 | @staticmethod 57 | def _parse_action(line, m, lines, *_): 58 | if m is None: 59 | raise ValueError(f'ed line contains invalid command: {line.rstrip()}') 60 | parts = m.groupdict() 61 | start = int(parts['start']) 62 | if start > 2147483647: 63 | raise ValueError(f'ed line contains line number > INT32_MAX: {start}') 64 | end = parts['end'] 65 | if end is None: 66 | end = start 67 | elif end == '$': 68 | end = '$' 69 | else: 70 | end = int(end) 71 | if end > 2147483647: 72 | raise ValueError(f'ed line contains line number > INT32_MAX: {end}') 73 | if end < start: 74 | raise ValueError(f'ed line contains invalid range: ({start}, {end})') 75 | action = EdActionType(parts['action']) 76 | 77 | data = [] 78 | if action in (EdActionType.Append, EdActionType.Change): 79 | for line in lines: 80 | if line == '.': 81 | break 82 | data.append(line) 83 | 84 | return EdAction(start, end, action, data) 85 | 86 | def __init__(self, out_name): 87 | super().__init__( 88 | r'(?P\d+)(?:,(?P\d+|\$))?(?P[acd])', self._parse_action, out_name, as_list=True 89 | ) 90 | 91 | 92 | class NetworkStatusDiffDocument(TorDocument): 93 | DOCUMENT_NAME = 'network_status_diff' 94 | 95 | # The first line is "network-status-diff-version 1" NL 96 | START_ITEM = ItemInt('network-status-diff-version') 97 | 98 | ITEMS = [ 99 | # The second line is "hash" SP FromDigest SP ToDigest NL 100 | Item('hash', parse_func=ItemParsers.split_symbol, parse_args=[' ', ['from_digest', 'to_digest']]), 101 | # Diff Actions 102 | ItemAction(out_name='actions'), 103 | ] 104 | 105 | def __init__(self, raw_string): 106 | super().__init__(raw_string) 107 | -------------------------------------------------------------------------------- /torpy/guard.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import logging 17 | import functools 18 | 19 | from torpy.cells import CellRelay, CellDestroy, CellCreatedFast, CellCreated2, CellRelayTruncated 20 | from torpy.utils import retry, log_retry 21 | from torpy.circuit import TorReceiver, CircuitsList, CellTimeoutError, CellHandlerManager, CircuitExtendError 22 | from torpy.cell_socket import TorCellSocket 23 | 24 | logger = logging.getLogger(__name__) 25 | 26 | 27 | class GuardState: 28 | Disconnecting = 1 29 | Disconnected = 2 30 | Connecting = 3 31 | Connected = 4 32 | 33 | 34 | def cell_to_circuit(func): 35 | def wrapped(_self, cell, *args, **kwargs): 36 | circuit = _self._circuits.get_by_id(cell.circuit_id) 37 | if not circuit: 38 | if _self._state != GuardState.Connected: 39 | logger.debug('Ignore not found circuits on %r state', _self._state) 40 | return 41 | raise Exception('Circuit #{:x} not found'.format(cell.circuit_id)) 42 | args_new = [_self, cell, circuit] + list(args) 43 | return func(*args_new, **kwargs) 44 | 45 | return wrapped 46 | 47 | 48 | class TorSender: 49 | def __init__(self, tor_socket): 50 | self._tor_socket = tor_socket 51 | 52 | def send(self, cell): 53 | self._tor_socket.send_cell(cell) 54 | 55 | 56 | class TorGuard: 57 | def __init__(self, router, purpose=None, consensus=None, auth_data=None): 58 | self._router = router 59 | self._purpose = purpose 60 | self._consensus = consensus 61 | self._auth_data = auth_data 62 | 63 | self._state = GuardState.Connecting 64 | logger.info('Connecting to guard node %s... (%s)', self._router, self._purpose) 65 | self.__tor_socket = TorCellSocket(self._router) 66 | self.__tor_socket.connect() 67 | 68 | self._sender = TorSender(self.__tor_socket) 69 | self._circuits = CircuitsList(self) 70 | 71 | self._handler_mgr = CellHandlerManager() 72 | self._handler_mgr.subscribe_for(CellDestroy, self._on_destroy) 73 | self._handler_mgr.subscribe_for([CellCreatedFast, CellCreated2], self._on_cell) 74 | self._handler_mgr.subscribe_for(CellRelay, self._on_relay) 75 | 76 | self._receiver = TorReceiver(self.__tor_socket, self._handler_mgr) 77 | self._receiver.start() 78 | 79 | self._state = GuardState.Connected 80 | 81 | @property 82 | def consensus(self): 83 | return self._consensus 84 | 85 | @property 86 | def router(self): 87 | return self._router 88 | 89 | @property 90 | def auth_data(self): 91 | return self._auth_data 92 | 93 | def __enter__(self): 94 | """Return Guard object.""" 95 | return self 96 | 97 | def __exit__(self, exc_type, exc_val, exc_tb): 98 | """Disconnect from Guard node.""" 99 | self.close() 100 | 101 | def close(self): 102 | logger.info('Closing guard connections (%s)...', self._purpose) 103 | self._state = GuardState.Disconnecting 104 | self._destroy_all_circuits() 105 | self._receiver.stop() 106 | self.__tor_socket.close() 107 | self._state = GuardState.Disconnected 108 | 109 | def _destroy_all_circuits(self): 110 | logger.debug('Destroying all circuits...') 111 | if self._circuits: 112 | for circuit in list(self._circuits.values()): 113 | self.destroy_circuit(circuit) 114 | 115 | @cell_to_circuit 116 | def _on_destroy(self, cell, circuit): 117 | logger.info('On destroy: circuit #%x', cell.circuit_id) 118 | send_destroy = isinstance(cell, CellRelayTruncated) 119 | self.destroy_circuit(circuit, send_destroy=send_destroy) 120 | 121 | @cell_to_circuit 122 | def _on_cell(self, cell, circuit): 123 | circuit.handle_cell(cell) 124 | 125 | @cell_to_circuit 126 | def _on_relay(self, cell: CellRelay, circuit): 127 | circuit.handle_relay(cell) 128 | 129 | def send_cell(self, cell): 130 | return self._sender.send(cell) 131 | 132 | @retry( 133 | 3, (CircuitExtendError, CellTimeoutError), log_func=functools.partial(log_retry, msg='Retry circuit creation') 134 | ) 135 | def create_circuit(self, hops_count, extend_routers=None): 136 | if self._state != GuardState.Connected: 137 | raise Exception('You must connect to guard node first') 138 | 139 | circuit = self._circuits.create_new() 140 | try: 141 | circuit.create() 142 | 143 | circuit.build_hops(hops_count) 144 | 145 | if extend_routers: 146 | for router in extend_routers: 147 | circuit.extend(router) 148 | except Exception: 149 | # We must close here because we didn't enter to circuit yet to guard by context manager 150 | circuit.close() 151 | raise 152 | 153 | return circuit 154 | 155 | def destroy_circuit(self, circuit, send_destroy=True): 156 | logger.info('Destroy circuit #%x', circuit.id) 157 | circuit.destroy(send_destroy=send_destroy) 158 | self._circuits.remove(circuit.id) 159 | 160 | def register(self, sock_or_stream, events, callback): 161 | return self._receiver.register(sock_or_stream, events, callback) 162 | 163 | def unregister(self, sock_or_stream): 164 | self._receiver.unregister(sock_or_stream) 165 | -------------------------------------------------------------------------------- /torpy/hiddenservice.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import math 17 | import time 18 | import struct 19 | import logging 20 | from base64 import b32decode, b32encode 21 | from typing import TYPE_CHECKING 22 | 23 | from torpy.cells import CellRelayRendezvous2 24 | from torpy.utils import AuthType 25 | from torpy.parsers import IntroPointParser, HSDescriptorParser 26 | from torpy.crypto_common import sha1, aes_update, aes_ctr_decryptor, b64decode, curve25519_public_from_bytes 27 | 28 | if TYPE_CHECKING: 29 | from torpy.circuit import TorCircuit 30 | 31 | logger = logging.getLogger(__name__) 32 | 33 | 34 | # tor ref: handle_control_hsfetch 35 | # tor ref: connection_ap_handle_onion 36 | class HiddenService: 37 | # Length of 'y' portion of 'y.onion' URL. 38 | REND_SERVICE_ID_LEN_BASE32 = 16 39 | # Length of a binary-encoded rendezvous service ID. 40 | REND_SERVICE_ID_LEN = 10 41 | 42 | # ... 43 | ED25519_PUBKEY_LEN = 32 44 | 45 | # The amount of bytes we use from the address checksum. 46 | HS_SERVICE_ADDR_CHECKSUM_LEN_USED = 2 47 | 48 | # Length of the binary encoded service address which is of course before the 49 | # base32 encoding. Construction is: 50 | # PUBKEY || CHECKSUM || VERSION 51 | # with 1 byte VERSION and 2 bytes CHECKSUM. The following is 35 bytes. 52 | HS_SERVICE_ADDR_LEN = (ED25519_PUBKEY_LEN + HS_SERVICE_ADDR_CHECKSUM_LEN_USED + 1) 53 | 54 | # Length of 'y' portion of 'y.onion' URL. This is base32 encoded and the 55 | # length ends up to 56 bytes (not counting the terminated NUL byte.) 56 | HS_SERVICE_ADDR_LEN_BASE32 = math.ceil(HS_SERVICE_ADDR_LEN * 8 / 5) 57 | 58 | HS_NO_AUTH = (None, AuthType.No) 59 | 60 | def __init__(self, onion_address, descriptor_cookie=None, auth_type=AuthType.No): 61 | self._onion_address, self._permanent_id, onion_identity_pk = self.parse_onion(onion_address) 62 | self._onion_identity_pk = curve25519_public_from_bytes(onion_identity_pk) if onion_identity_pk else None 63 | if self._onion_identity_pk: 64 | raise Exception('v3 onion hidden service not supported yet') 65 | self._descriptor_cookie = b64decode(descriptor_cookie) if descriptor_cookie else None 66 | self._auth_type = auth_type 67 | if descriptor_cookie and auth_type == AuthType.No: 68 | raise RuntimeError('You must specify auth type') 69 | if not descriptor_cookie and auth_type != AuthType.No: 70 | raise RuntimeError('You must specify descriptor cookie') 71 | 72 | @staticmethod 73 | def normalize_onion(onion_address): 74 | if onion_address.endswith('.onion'): 75 | onion_address = onion_address[:-6].rsplit('.', 1)[-1] 76 | 77 | if len(onion_address) != HiddenService.REND_SERVICE_ID_LEN_BASE32 and \ 78 | len(onion_address) != HiddenService.HS_SERVICE_ADDR_LEN_BASE32: 79 | raise Exception(f'Unknown onion address: {onion_address}') 80 | 81 | return onion_address 82 | 83 | @staticmethod 84 | def parse_onion(onion_address): 85 | onion_address = HiddenService.normalize_onion(onion_address) 86 | 87 | if len(onion_address) == HiddenService.REND_SERVICE_ID_LEN_BASE32: 88 | permanent_id = b32decode(onion_address.upper()) 89 | assert len(permanent_id) == HiddenService.REND_SERVICE_ID_LEN, 'You must specify valid V2 onion hostname' 90 | return onion_address, permanent_id, None 91 | elif len(onion_address) == HiddenService.HS_SERVICE_ADDR_LEN_BASE32: 92 | # tor ref: hs_parse_address 93 | decoded = b32decode(onion_address.upper()) 94 | pubkey = decoded[:HiddenService.ED25519_PUBKEY_LEN] 95 | # checksum decoded[self.ED25519_PUBKEY_LEN:self.ED25519_PUBKEY_LEN + self.HS_SERVICE_ADDR_CHECKSUM_LEN_USED] 96 | # version decoded[self.ED25519_PUBKEY_LEN + self.HS_SERVICE_ADDR_CHECKSUM_LEN_USED:] 97 | return onion_address, None, pubkey 98 | # fetch_v3_desc 99 | # pick_hsdir_v3 100 | # directory_launch_v3_desc_fetch 101 | 102 | @property 103 | def onion(self): 104 | return self._onion_address 105 | 106 | @property 107 | def hostname(self): 108 | return self._onion_address + '.onion' 109 | 110 | @property 111 | def permanent_id(self): 112 | """service-id or permanent-id.""" 113 | return self._permanent_id 114 | 115 | @property 116 | def descriptor_cookie(self): 117 | return self._descriptor_cookie 118 | 119 | @property 120 | def auth_type(self): 121 | return self._auth_type 122 | 123 | def _get_secret_id(self, replica): 124 | """ 125 | Get secret_id by replica number. 126 | 127 | rend-spec.txt 128 | 1.3. 129 | 130 | "time-period" changes periodically as a function of time and 131 | "permanent-id". The current value for "time-period" can be calculated 132 | using the following formula: 133 | 134 | time-period = (current-time + permanent-id-byte * 86400 / 256) 135 | / 86400 136 | """ 137 | # tor ref: get_secret_id_part_bytes 138 | permanent_byte = self._permanent_id[0] 139 | time_period = int((int(time.time()) + (permanent_byte * 86400 / 256)) / 86400) 140 | if self._descriptor_cookie and self._auth_type == AuthType.Stealth: 141 | buff = struct.pack('!I16sB', time_period, self._descriptor_cookie, replica) 142 | else: 143 | buff = struct.pack('!IB', time_period, replica) 144 | return sha1(buff) 145 | 146 | def get_descriptor_id(self, replica): 147 | # tor ref: rend_compute_v2_desc_id 148 | # Calculate descriptor ID: H(permanent-id | secret-id-part) 149 | buff = self._permanent_id + self._get_secret_id(replica) 150 | return sha1(buff) 151 | 152 | 153 | class HiddenServiceConnector: 154 | def __init__(self, circuit, consensus): 155 | self._circuit = circuit 156 | self._consensus = consensus 157 | 158 | def get_responsibles_dir(self, hidden_service): 159 | for i, responsible_router in enumerate(self._consensus.get_responsibles(hidden_service)): 160 | replica = 1 if i >= 3 else 0 161 | yield ResponsibleDir(responsible_router, replica, self._circuit, self._consensus) 162 | 163 | 164 | class EncPointsBuffer: 165 | # /** Length of our symmetric cipher's keys of 128-bit. */ 166 | CIPHER_KEY_LEN = 16 167 | # /** Length of our symmetric cipher's IV of 128-bit. */ 168 | CIPHER_IV_LEN = 16 169 | # /** Length of our symmetric cipher's keys of 256-bit. */ 170 | CIPHER256_KEY_LEN = 32 171 | # /** Length of client identifier in encrypted introduction points for hidden 172 | # * service authorization type 'basic'. */ 173 | REND_BASIC_AUTH_CLIENT_ID_LEN = 4 174 | # /** Multiple of the number of clients to which the real number of clients 175 | # * is padded with fake clients for hidden service authorization type 176 | # * 'basic'. */ 177 | REND_BASIC_AUTH_CLIENT_MULTIPLE = 16 178 | # /** Length of client entry consisting of client identifier and encrypted 179 | # * session key for hidden service authorization type 'basic'. */ 180 | REND_BASIC_AUTH_CLIENT_ENTRY_LEN = REND_BASIC_AUTH_CLIENT_ID_LEN + CIPHER_KEY_LEN 181 | 182 | def __init__(self, crypted_data): 183 | self._crypted_data = crypted_data 184 | assert len(crypted_data) > 2, 'Size of crypted data too small' 185 | self._auth_type = int(crypted_data[0]) 186 | # fmt: off 187 | self._auth_to_func = {AuthType.Basic: self._decrypt_basic, 188 | AuthType.Stealth: self._decrypt_stealth} 189 | # fmt: on 190 | 191 | @property 192 | def auth_type(self): 193 | return self._auth_type 194 | 195 | def decrypt(self, descriptor_cookie): 196 | # tor ref: rend_decrypt_introduction_points 197 | return self._auth_to_func[self._auth_type](descriptor_cookie) 198 | 199 | def _decrypt_basic(self, descriptor_cookie): 200 | assert self._crypted_data[0] == AuthType.Basic 201 | block_count = self._crypted_data[1] 202 | entries_len = block_count * self.REND_BASIC_AUTH_CLIENT_MULTIPLE * self.REND_BASIC_AUTH_CLIENT_ENTRY_LEN 203 | assert len(self._crypted_data) > 2 + entries_len + self.CIPHER_IV_LEN, 'Size of crypted data too small' 204 | iv = self._crypted_data[2 + entries_len:2 + entries_len + self.CIPHER_IV_LEN] 205 | client_id = sha1(descriptor_cookie + iv)[:4] 206 | session_key = self._get_session_key(self._crypted_data[2:2 + entries_len], descriptor_cookie, client_id) 207 | d = aes_ctr_decryptor(session_key, iv) 208 | data = self._crypted_data[2 + entries_len + self.CIPHER_IV_LEN:] 209 | return d.update(data) 210 | 211 | def _get_session_key(self, data, descriptor_cookie, client_id): 212 | pos = 0 213 | d = aes_ctr_decryptor(descriptor_cookie) 214 | while pos < len(data): 215 | if data[pos:pos + self.REND_BASIC_AUTH_CLIENT_ID_LEN] == client_id: 216 | start_key_pos = pos + self.REND_BASIC_AUTH_CLIENT_ID_LEN 217 | end_key_pos = start_key_pos + self.CIPHER_KEY_LEN 218 | enc_session_key = data[start_key_pos:end_key_pos] 219 | return aes_update(d, enc_session_key) 220 | pos += self.REND_BASIC_AUTH_CLIENT_ENTRY_LEN 221 | raise Exception('Session key for client {!r} not found'.format(client_id)) 222 | 223 | def _decrypt_stealth(self, descriptor_cookie): 224 | assert len(self._crypted_data) > 2 + self.CIPHER_IV_LEN, 'Size of encrypted data is too small' 225 | assert self._crypted_data[0] == AuthType.Stealth 226 | iv = self._crypted_data[1:1 + self.CIPHER_IV_LEN] 227 | d = aes_ctr_decryptor(descriptor_cookie, iv) 228 | data = self._crypted_data[1 + self.CIPHER_IV_LEN:] 229 | return d.update(data) 230 | 231 | 232 | class DescriptorNotAvailable(Exception): 233 | """Descriptor not found.""" 234 | 235 | 236 | class ResponsibleDir: 237 | def __init__(self, router, replica, circuit, consensus): 238 | self._router = router 239 | self._replica = replica 240 | self._circuit = circuit 241 | self._consensus = consensus 242 | 243 | @property 244 | def replica(self): 245 | return self._replica 246 | 247 | def get_introductions(self, hidden_service): 248 | descriptor_id = hidden_service.get_descriptor_id(self.replica) 249 | response = self._fetch_descriptor(descriptor_id) 250 | for intro_point in self._get_intro_points(response, hidden_service.descriptor_cookie): 251 | yield intro_point 252 | 253 | def _fetch_descriptor(self, descriptor_id): 254 | # tor ref: rend_client_fetch_v2_desc 255 | # tor ref: fetch_v3_desc 256 | 257 | logger.info('Create circuit for hsdir') 258 | with self._circuit.create_new_circuit(extend_routers=[self._router]) as directory_circuit: 259 | assert directory_circuit.nodes_count == 2 260 | 261 | with directory_circuit.create_dir_client() as dir_client: 262 | # tor ref: directory_send_command (DIR_PURPOSE_FETCH_RENDDESC_V2) 263 | descriptor_id_str = b32encode(descriptor_id).decode().lower() 264 | descriptor_path = f'/tor/rendezvous2/{descriptor_id_str}' 265 | 266 | status, response = dir_client.get(descriptor_path) 267 | response = response.decode() 268 | if status != 200: 269 | logger.error('No valid response from hsdir. Status = %r. Body: %r', status, response) 270 | raise DescriptorNotAvailable("Couldn't fetch descriptor") 271 | 272 | return response 273 | 274 | def _info_to_router(self, intro_point_info): 275 | onion_router = self._consensus.get_router(intro_point_info['introduction_point']) 276 | onion_router.service_key = intro_point_info['service_key'] 277 | onion_router.onion_key = intro_point_info['onion_key'] 278 | return onion_router 279 | 280 | def _get_intro_points(self, response, descriptor_cookie): 281 | intro_points_raw_base64 = HSDescriptorParser.parse(response) 282 | intro_points_raw = b64decode(intro_points_raw_base64) 283 | 284 | # Check whether it's encrypted 285 | if intro_points_raw[0] == AuthType.Basic or intro_points_raw[0] == AuthType.Stealth: 286 | if not descriptor_cookie: 287 | raise Exception('Hidden service needs descriptor_cookie for authorization') 288 | enc_buff = EncPointsBuffer(intro_points_raw) 289 | intro_points_raw = enc_buff.decrypt(descriptor_cookie) 290 | elif descriptor_cookie: 291 | logger.warning("Descriptor cookie was specified but hidden service hasn't encrypted intro points") 292 | 293 | if not intro_points_raw.startswith(b'introduction-point '): 294 | raise Exception('Unknown introduction point data received') 295 | 296 | intro_points_raw = intro_points_raw.decode() 297 | intro_points_info_list = IntroPointParser.parse(intro_points_raw) 298 | 299 | for intro_point_info in intro_points_info_list: 300 | router = self._info_to_router(intro_point_info) 301 | yield IntroductionPoint(router, self._circuit) 302 | 303 | def __str__(self): 304 | """Format ResponsibleDir string representation.""" 305 | return 'ResponsibleDir {}'.format(self._router) 306 | 307 | 308 | class IntroductionPoint: 309 | def __init__(self, router, circuit: 'TorCircuit'): 310 | self._introduction_router = router 311 | self._circuit = circuit 312 | 313 | def connect(self, hidden_service, rendezvous_cookie): 314 | # Waiting for CellRelayRendezvous2 in our main circuit 315 | with self._circuit.create_waiter(CellRelayRendezvous2) as w: 316 | # Create introduction point circuit 317 | with self._circuit.create_new_circuit(extend_routers=[self._introduction_router]) as intro_circuit: 318 | assert intro_circuit.nodes_count == 2 319 | 320 | # TODO: tor ref: v3 hs_client send_introduce1 321 | # Send Introduce1 322 | extend_node = intro_circuit.rendezvous_introduce( 323 | self._circuit, 324 | rendezvous_cookie, 325 | hidden_service.auth_type, 326 | hidden_service.descriptor_cookie, 327 | ) 328 | 329 | rendezvous2_cell = w.get(timeout=10) 330 | extend_node.complete_handshake(rendezvous2_cell.handshake_data) 331 | return extend_node 332 | -------------------------------------------------------------------------------- /torpy/http/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/torpyorg/torpy/ebf000cc93d2d7b3ddf68f2604afd46c998ae1c1/torpy/http/__init__.py -------------------------------------------------------------------------------- /torpy/http/adapter.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import logging 17 | 18 | from requests.adapters import DEFAULT_POOLBLOCK, HTTPAdapter 19 | try: 20 | from requests.packages.urllib3.connection import HTTPConnection, VerifiedHTTPSConnection 21 | from requests.packages.urllib3.exceptions import NewConnectionError, ConnectTimeoutError 22 | from requests.packages.urllib3.poolmanager import PoolManager, HTTPConnectionPool, HTTPSConnectionPool 23 | except ImportError: 24 | # requests >=2.16 25 | from urllib3.connection import HTTPConnection, VerifiedHTTPSConnection 26 | from urllib3.exceptions import NewConnectionError, ConnectTimeoutError 27 | from urllib3.poolmanager import PoolManager, HTTPConnectionPool, HTTPSConnectionPool 28 | 29 | from torpy.http.base import TorInfo 30 | 31 | logger = logging.getLogger(__name__) 32 | 33 | 34 | class TorHttpAdapter(HTTPAdapter): 35 | def __init__(self, guard, hops_count, retries=0): 36 | self._tor_info = TorInfo(guard, hops_count) 37 | super().__init__(max_retries=retries) 38 | 39 | def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs): 40 | # save these values for pickling 41 | self._pool_connections = connections 42 | self._pool_maxsize = maxsize 43 | self._pool_block = block 44 | 45 | self.poolmanager = MyPoolManager( 46 | self._tor_info, num_pools=connections, maxsize=maxsize, block=block, strict=True, **pool_kwargs 47 | ) 48 | 49 | 50 | def wrap_normalizer(base_normalizer): 51 | def wrapped(request_context, *args, **kwargs): 52 | context = request_context.copy() 53 | context.pop('tor_info') 54 | return base_normalizer(context, *args, **kwargs) 55 | return wrapped 56 | 57 | 58 | class MyPoolManager(PoolManager): 59 | def __init__(self, tor_info, *args, **kwargs): 60 | super().__init__(*args, **kwargs) 61 | self.pool_classes_by_scheme = { 62 | 'http': MyHTTPConnectionPool, 63 | 'https': MyHTTPSConnectionPool, 64 | } 65 | # for requests >= 2.11 66 | if hasattr(self, 'key_fn_by_scheme'): 67 | self.key_fn_by_scheme = { 68 | 'http': wrap_normalizer(self.key_fn_by_scheme['http']), 69 | 'https': wrap_normalizer(self.key_fn_by_scheme['https']), 70 | } 71 | self.connection_pool_kw['tor_info'] = tor_info 72 | 73 | 74 | class MyHTTPConnectionPool(HTTPConnectionPool): 75 | def __init__(self, *args, **kwargs): 76 | self._tor_info = kwargs.pop('tor_info') 77 | super().__init__(*args, **kwargs) 78 | 79 | def _new_conn(self): 80 | self.num_connections += 1 81 | logger.debug('[MyHTTPConnectionPool] new_conn %i', self.num_connections) 82 | circuit = self._tor_info.get_circuit(self.host) 83 | return MyHTTPConnection( 84 | circuit, 85 | host=self.host, 86 | port=self.port, 87 | timeout=self.timeout.connect_timeout, 88 | strict=self.strict, 89 | **self.conn_kw, 90 | ) 91 | 92 | 93 | class MyHTTPSConnectionPool(HTTPSConnectionPool): 94 | def __init__(self, *args, **kwargs): 95 | self._tor_info = kwargs.pop('tor_info') 96 | super().__init__(*args, **kwargs) 97 | 98 | def _new_conn(self): 99 | self.num_connections += 1 100 | logger.debug('[MyHTTPSConnectionPool] new_conn %i', self.num_connections) 101 | circuit = self._tor_info.get_circuit(self.host) 102 | conn = MyHTTPSConnection( 103 | circuit, 104 | host=self.host, 105 | port=self.port, 106 | timeout=self.timeout.connect_timeout, 107 | strict=self.strict, 108 | **self.conn_kw, 109 | ) 110 | logger.debug('[MyHTTPSConnectionPool] preparing...') 111 | return self._prepare_conn(conn) 112 | # TODO: override close to close all circuits? 113 | 114 | 115 | class MyHTTPConnection(HTTPConnection): 116 | def __init__(self, circuit, *args, **kwargs): 117 | self._circuit = circuit 118 | self._tor_stream = None 119 | super().__init__(*args, **kwargs) 120 | 121 | def connect(self): 122 | logger.debug('[MyHTTPConnection] connect %s:%i', self.host, self.port) 123 | try: 124 | self._tor_stream = self._circuit.create_stream((self.host, self.port)) 125 | logger.debug('[MyHTTPConnection] tor_stream create_socket') 126 | self.sock = self._tor_stream.create_socket() 127 | if self._tunnel_host: 128 | self._tunnel() 129 | except TimeoutError: 130 | logger.error('TimeoutError') 131 | raise ConnectTimeoutError( 132 | self, 'Connection to %s timed out. (connect timeout=%s)' % (self.host, self.timeout) 133 | ) 134 | except Exception as e: 135 | logger.error('NewConnectionError') 136 | raise NewConnectionError(self, 'Failed to establish a new connection: %s' % e) 137 | 138 | def close(self): 139 | # WARN: self.sock will be closed inside base class 140 | logger.debug('[MyHTTPConnection] closing') 141 | super().close() 142 | logger.debug('[MyHTTPConnection] circuit destroy_stream') 143 | if self._tor_stream: 144 | self._tor_stream.close() 145 | logger.debug('[MyHTTPConnection] closed') 146 | 147 | 148 | class MyHTTPSConnection(VerifiedHTTPSConnection): 149 | def __init__(self, circuit, *args, **kwargs): 150 | self._circuit = circuit 151 | self._tor_stream = None 152 | super().__init__(*args, **kwargs) 153 | 154 | def _new_conn(self): 155 | logger.debug('[MyHTTPSConnection] new conn %s:%i', self.host, self.port) 156 | try: 157 | self._tor_stream = self._circuit.create_stream((self.host, self.port)) 158 | logger.debug('[MyHTTPSConnection] tor_stream create_socket') 159 | return self._tor_stream.create_socket() 160 | except TimeoutError: 161 | logger.error('TimeoutError') 162 | raise ConnectTimeoutError( 163 | self, 'Connection to %s timed out. (connect timeout=%s)' % (self.host, self.timeout) 164 | ) 165 | except Exception as e: 166 | logger.error('NewConnectionError') 167 | raise NewConnectionError(self, 'Failed to establish a new connection: %s' % e) 168 | 169 | def close(self): 170 | logger.debug('[MyHTTPSConnection] closing %s', self.host) 171 | super().close() 172 | logger.debug('[MyHTTPSConnection] circuit destroy_stream') 173 | if self._tor_stream: 174 | self._tor_stream.close() 175 | logger.debug('[MyHTTPSConnection] closed') 176 | -------------------------------------------------------------------------------- /torpy/http/base.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import logging 3 | 4 | from torpy.utils import hostname_key 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | class SocketProxy: 10 | def __init__(self, sock, tor_stream): 11 | self._sock = sock 12 | self._tor_stream = tor_stream 13 | 14 | def __getattr__(self, attr): 15 | """Proxying methods to real socket.""" 16 | if attr in self.__dict__: 17 | return getattr(self, attr) 18 | return getattr(self._sock, attr) 19 | 20 | @classmethod 21 | def rewrap(cls, prev_proxy, new_sock): 22 | return cls(new_sock, prev_proxy.tor_stream) 23 | 24 | @property 25 | def wrapped_sock(self): 26 | return self._sock 27 | 28 | @property 29 | def tor_stream(self): 30 | return self._tor_stream 31 | 32 | def close(self): 33 | logger.debug('[SocketProxy] close') 34 | self.close_tor_stream() 35 | self._sock.close() 36 | 37 | def close_tor_stream(self): 38 | self._tor_stream.close() 39 | 40 | 41 | class TorInfo: 42 | def __init__(self, guard, hops_count): 43 | self._guard = guard 44 | self._hops_count = hops_count 45 | self._circuits = {} 46 | self._lock = threading.Lock() 47 | 48 | def get_circuit(self, hostname): 49 | host_key = hostname_key(hostname) 50 | logger.debug('[TorInfo] Waiting lock...') 51 | with self._lock: 52 | logger.debug('[TorInfo] Got lock...') 53 | circuit = self._circuits.get(host_key) 54 | if not circuit: 55 | logger.debug('[TorInfo] Create new circuit for %s (key %s)', hostname, host_key) 56 | circuit = self._guard.create_circuit(self._hops_count) 57 | self._circuits[host_key] = circuit 58 | else: 59 | logger.debug('[TorInfo] Use existing...') 60 | return circuit 61 | 62 | def connect(self, address, timeout=30, source_address=None): 63 | circuit = self.get_circuit(address[0]) 64 | tor_stream = circuit.create_stream(address) 65 | logger.debug('[TorHTTPConnection] tor_stream create_socket') 66 | return SocketProxy(tor_stream.create_socket(), tor_stream) 67 | -------------------------------------------------------------------------------- /torpy/http/client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import gzip 17 | import zlib 18 | import logging 19 | from io import BytesIO 20 | from http.client import parse_headers 21 | 22 | from torpy.utils import recv_all 23 | 24 | logger = logging.getLogger(__name__) 25 | 26 | 27 | class HttpStreamClient: 28 | def __init__(self, stream, host=None): 29 | self._stream = stream 30 | self._host = host 31 | 32 | def get(self, path, host=None, headers: dict = None): 33 | headers = headers or {} 34 | host = host or self._host 35 | if host: 36 | headers['Host'] = host 37 | headers_str = '\r\n'.join(f'{key}: {val}' for (key, val) in headers.items()) 38 | http_query = f'GET {path} HTTP/1.0\r\n{headers_str}\r\n\r\n' 39 | self._stream.send(http_query.encode()) 40 | 41 | raw_response = recv_all(self._stream) 42 | header, body = raw_response.split(b'\r\n\r\n', 1) 43 | 44 | f = BytesIO(header) 45 | request_line = f.readline().split(b' ') 46 | protocol, status = request_line[:2] 47 | status = int(status) 48 | 49 | headers = parse_headers(f) 50 | if headers['Content-Encoding'] == 'deflate': 51 | body = zlib.decompress(body) 52 | elif headers['Content-Encoding'] == 'gzip': 53 | body = gzip.decompress(body) 54 | 55 | if status != 200: 56 | logger.debug('raw_response = %s', raw_response) 57 | 58 | return status, body 59 | 60 | def close(self): 61 | self._stream.close() 62 | 63 | def __enter__(self): 64 | """Start using the http client.""" 65 | return self 66 | 67 | def __exit__(self, exc_type, exc_val, exc_tb): 68 | """Close the http client.""" 69 | self.close() 70 | -------------------------------------------------------------------------------- /torpy/http/requests.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import logging 17 | from contextlib import contextmanager 18 | 19 | from requests import Request, Session 20 | 21 | try: 22 | from urllib3.util import SKIP_HEADER 23 | except Exception: 24 | SKIP_HEADER = None 25 | 26 | from torpy.client import TorClient 27 | from torpy.http.adapter import TorHttpAdapter 28 | 29 | logger = logging.getLogger(__name__) 30 | 31 | 32 | class TorRequests: 33 | def __init__(self, hops_count=3, headers=None, auth_data=None): 34 | self._hops_count = hops_count 35 | self._headers = dict(headers) if headers else {} 36 | self._auth_data = dict(auth_data) if auth_data else auth_data 37 | 38 | def __enter__(self): 39 | """Create TorClient and connect to guard node.""" 40 | self._tor = TorClient(auth_data=self._auth_data) 41 | self._guard = self._tor.get_guard() 42 | return self 43 | 44 | def __exit__(self, exc_type, exc_val, exc_tb): 45 | """Close guard connection.""" 46 | self._guard.close() 47 | self._tor.close() 48 | 49 | def send(self, method, url, data=None, **kwargs): 50 | with self.get_session() as s: 51 | r = Request(method, url, data, **kwargs) 52 | return s.send(r.prepare()) 53 | 54 | @contextmanager 55 | def get_session(self, retries=0): 56 | adapter = TorHttpAdapter(self._guard, self._hops_count, retries=retries) 57 | with Session() as s: 58 | s.headers.update(self._headers) 59 | s.mount('http://', adapter) 60 | s.mount('https://', adapter) 61 | yield s 62 | 63 | 64 | @contextmanager 65 | def tor_requests_session(hops_count=3, headers=None, auth_data=None, retries=0): 66 | with TorRequests(hops_count, headers, auth_data) as tr: 67 | with tr.get_session(retries=retries) as s: 68 | yield s 69 | 70 | 71 | def do_request(url, method='GET', data=None, headers=None, hops=3, auth_data=None, verbose=0, retries=0): 72 | with tor_requests_session(hops, auth_data, retries=retries) as s: 73 | headers = dict(headers or []) 74 | # WARN: https://github.com/urllib3/urllib3/pull/1750 75 | if SKIP_HEADER and \ 76 | 'user-agent' not in (k.lower() for k in headers.keys()): 77 | headers['User-Agent'] = SKIP_HEADER 78 | request = Request(method, url, data=data, headers=headers) 79 | logger.warning('Sending: %s %s', request.method, request.url) 80 | response = s.send(request.prepare()) 81 | logger.warning('Response status: %r', response.status_code) 82 | return response.text 83 | -------------------------------------------------------------------------------- /torpy/http/urlopener.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import logging 17 | import socket 18 | from typing import ContextManager 19 | from contextlib import contextmanager 20 | from http.client import HTTPConnection, HTTPSConnection, HTTPResponse 21 | from urllib.error import URLError 22 | from urllib.request import ( 23 | Request, 24 | OpenerDirector, 25 | ProxyHandler, 26 | UnknownHandler, 27 | HTTPRedirectHandler, 28 | HTTPDefaultErrorHandler, 29 | HTTPErrorProcessor, 30 | HTTPHandler, 31 | HTTPSHandler, 32 | ) 33 | 34 | from torpy.http.base import TorInfo, SocketProxy 35 | from torpy import TorClient 36 | 37 | logger = logging.getLogger(__name__) 38 | 39 | 40 | class TorHTTPResponse(HTTPResponse): 41 | def __init__(self, sock, debuglevel=0, method=None, url=None): 42 | logger.debug('[TorHTTPResponse] init') 43 | super().__init__(sock, debuglevel=debuglevel, method=method, url=url) 44 | self._sock = sock 45 | 46 | def close(self): 47 | self._sock.close_tor_stream() 48 | super().close() 49 | 50 | 51 | class TorHTTPConnection(HTTPConnection): 52 | response_class = TorHTTPResponse 53 | # debuglevel = 1 54 | 55 | def __init__(self, *args, **kwargs): 56 | self._tor_info = kwargs.pop('tor_info') 57 | super().__init__(*args, **kwargs) 58 | 59 | def connect(self): 60 | """Connect to the host and port specified in __init__.""" 61 | self.sock = self._tor_info.connect((self.host, self.port), self.timeout, self.source_address) 62 | 63 | if self._tunnel_host: 64 | self._tunnel() 65 | 66 | def close(self): 67 | logger.debug('[TorHTTPConnection] close') 68 | super().close() 69 | 70 | 71 | class TorHTTPSConnection(TorHTTPConnection, HTTPSConnection): 72 | def __init__(self, *args, **kwargs): 73 | super().__init__(*args, **kwargs) 74 | 75 | def connect(self): 76 | TorHTTPConnection.connect(self) 77 | 78 | if self._tunnel_host: 79 | server_hostname = self._tunnel_host 80 | else: 81 | server_hostname = self.host 82 | 83 | ssl_sock = self._context.wrap_socket(self.sock.wrapped_sock, server_hostname=server_hostname) 84 | self.sock = SocketProxy.rewrap(self.sock, ssl_sock) 85 | 86 | 87 | class TorHTTPHandler(HTTPHandler): 88 | def __init__(self, guard, hops_count, debuglevel=0): 89 | super().__init__(debuglevel=debuglevel) 90 | self._tor_info = TorInfo(guard, hops_count) 91 | 92 | def http_open(self, req): 93 | return self.do_open(TorHTTPConnection, req, tor_info=self._tor_info) 94 | 95 | 96 | class TorHTTPSHandler(HTTPSHandler): 97 | def __init__(self, guard, hops_count, debuglevel=0, context=None, check_hostname=None): 98 | super().__init__(debuglevel=debuglevel, context=context, check_hostname=check_hostname) 99 | self._tor_info = TorInfo(guard, hops_count) 100 | 101 | def https_open(self, req): 102 | return self.do_open(TorHTTPSConnection, req, 103 | context=self._context, check_hostname=self._check_hostname, tor_info=self._tor_info) 104 | 105 | 106 | class RetryOpenerDirector(OpenerDirector): 107 | def open(self, fullurl, retries=1, *args, **kwargs): 108 | assert retries >= 1 109 | last_err = None 110 | for _ in range(retries): 111 | try: 112 | return super().open(fullurl, *args, **kwargs) 113 | except URLError as err: 114 | last_err = err 115 | if not isinstance(err.reason, socket.timeout): 116 | raise 117 | else: 118 | raise last_err 119 | 120 | 121 | def build_tor_opener(guard, hops_count=3, debuglevel=0): 122 | opener = RetryOpenerDirector() 123 | default_classes = [ProxyHandler, UnknownHandler, 124 | HTTPDefaultErrorHandler, HTTPRedirectHandler, 125 | HTTPErrorProcessor] 126 | for cls in default_classes: 127 | opener.add_handler(cls()) 128 | opener.add_handler(TorHTTPHandler(guard, hops_count, debuglevel=debuglevel)) 129 | opener.add_handler(TorHTTPSHandler(guard, hops_count, debuglevel=debuglevel)) 130 | opener.addheaders = [] 131 | return opener 132 | 133 | 134 | @contextmanager 135 | def tor_opener(hops_count=3, debuglevel=0, auth_data=None) -> ContextManager[RetryOpenerDirector]: 136 | with TorClient(auth_data=auth_data) as tor: 137 | with tor.get_guard() as guard: 138 | yield build_tor_opener(guard, hops_count=hops_count, debuglevel=debuglevel) 139 | 140 | 141 | def do_request(url, method='GET', data=None, headers=None, hops=3, auth_data=None, verbose=0, retries=3): 142 | with tor_opener(hops_count=hops, auth_data=auth_data, debuglevel=verbose) as opener: 143 | request = Request(url, data, method=method, headers=dict(headers or [])) 144 | 145 | logger.warning('Sending: %s %s', request.get_method(), request.full_url) 146 | with opener.open(request, retries=retries) as response: 147 | logger.warning('Response status: %r', response.status) 148 | logger.debug('Reading...') 149 | return response.read().decode('utf-8') 150 | -------------------------------------------------------------------------------- /torpy/keyagreement.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import os 17 | import logging 18 | from abc import ABCMeta, abstractmethod 19 | 20 | from torpy.utils import cached_property 21 | from torpy.crypto import TOR_DIGEST_LEN, kdf_tor 22 | from torpy.crypto_common import ( 23 | hmac, 24 | dh_public, 25 | dh_shared, 26 | dh_private, 27 | hkdf_sha256, 28 | curve25519_private, 29 | dh_public_to_bytes, 30 | curve25519_to_bytes, 31 | dh_public_from_bytes, 32 | curve25519_get_shared, 33 | curve25519_public_from_bytes, 34 | curve25519_public_from_private, 35 | ) 36 | 37 | logger = logging.getLogger(__name__) 38 | 39 | 40 | class KeyAgreementError(Exception): 41 | pass 42 | 43 | 44 | class KeyAgreement(metaclass=ABCMeta): 45 | # tor ref: CPATH_KEY_MATERIAL_LEN 46 | KEY_MATERIAL_LENGTH = 20 * 2 + 16 * 2 47 | 48 | # tor ref: or.h 49 | # #define ONION_HANDSHAKE_TYPE_TAP 0x0000 50 | # #define ONION_HANDSHAKE_TYPE_FAST 0x0001 51 | # #define ONION_HANDSHAKE_TYPE_NTOR 0x0002 52 | TYPE = -1 53 | 54 | def __init__(self, onion_router): 55 | pass 56 | 57 | @cached_property 58 | @abstractmethod 59 | def handshake(self): 60 | pass 61 | 62 | @abstractmethod 63 | def complete_handshake(self, handshake_response): 64 | pass 65 | 66 | 67 | class TapKeyAgreement(KeyAgreement): 68 | TYPE = 0 69 | # 70 | # 5.1.3. The "TAP" handshake 71 | # 72 | # This handshake uses Diffie-Hellman in Z_p and RSA to compute a set of 73 | # shared keys which the client knows are shared only with a particular 74 | # server, and the server knows are shared with whomever sent the 75 | # original handshake (or with nobody at all). It's not very fast and 76 | # not very good. (See Goldberg's "On the Security of the Tor 77 | # Authentication Protocol".) 78 | # 79 | # Define TAP_C_HANDSHAKE_LEN as DH_LEN+KEY_LEN+PK_PAD_LEN. 80 | # Define TAP_S_HANDSHAKE_LEN as DH_LEN+HASH_LEN. 81 | # 82 | # The payload for a CREATE cell is an 'onion skin', which consists of 83 | # the first step of the DH handshake data (also known as g^x). This 84 | # value is hybrid-encrypted (see 0.3) to the server's onion key, giving 85 | # a client handshake of: 86 | # 87 | # PK-encrypted: 88 | # Padding [PK_PAD_LEN bytes] 89 | # Symmetric key [KEY_LEN bytes] 90 | # First part of g^x [PK_ENC_LEN-PK_PAD_LEN-KEY_LEN bytes] 91 | # Symmetrically encrypted: 92 | # Second part of g^x [DH_LEN-(PK_ENC_LEN-PK_PAD_LEN-KEY_LEN) 93 | # bytes] 94 | # 95 | # The payload for a CREATED cell, or the relay payload for an 96 | # EXTENDED cell, contains: 97 | # DH data (g^y) [DH_LEN bytes] 98 | # Derivative key data (KH) [HASH_LEN bytes] 99 | # 100 | # Once the handshake between the OP and an OR is completed, both can 101 | # now calculate g^xy with ordinary DH. Before computing g^xy, both parties 102 | # MUST verify that the received g^x or g^y value is not degenerate; 103 | # that is, it must be strictly greater than 1 and strictly less than p-1 104 | # where p is the DH modulus. Implementations MUST NOT complete a handshake 105 | # with degenerate keys. Implementations MUST NOT discard other "weak" 106 | # g^x values. 107 | # 108 | # (Discarding degenerate keys is critical for security; if bad keys 109 | # are not discarded, an attacker can substitute the OR's CREATED 110 | # cell's g^y with 0 or 1, thus creating a known g^xy and impersonating 111 | # the OR. Discarding other keys may allow attacks to learn bits of 112 | # the private key.) 113 | # 114 | # Once both parties have g^xy, they derive their shared circuit keys 115 | # and 'derivative key data' value via the KDF-TOR function in 5.2.1. 116 | # 117 | 118 | def __init__(self, onion_router): 119 | super().__init__(onion_router) 120 | 121 | self._private_key = dh_private() 122 | self._public_key = dh_public(self._private_key) 123 | 124 | @cached_property 125 | def handshake(self): 126 | return dh_public_to_bytes(self._public_key) 127 | 128 | def complete_handshake(self, handshake_response): 129 | peer_pub_key_bytes = handshake_response[:128] 130 | auth = handshake_response[128:] # tap auth is SHA1, 20 in bytes? 131 | assert len(auth) == TOR_DIGEST_LEN, 'received wrong sha1 len' 132 | 133 | peer_pub_key = dh_public_from_bytes(peer_pub_key_bytes) 134 | shared_secret = dh_shared(self._private_key, peer_pub_key) 135 | computed_auth, key_material = kdf_tor(shared_secret) 136 | if computed_auth != auth: 137 | raise KeyAgreementError('Auth input does not match.') 138 | 139 | # Cut unused bytes 140 | return key_material[:KeyAgreement.KEY_MATERIAL_LENGTH] 141 | 142 | 143 | class FastKeyAgreement(KeyAgreement): 144 | TYPE = 1 145 | 146 | def __init__(self, onion_router): 147 | super().__init__(onion_router) 148 | 149 | @cached_property 150 | def handshake(self): 151 | # tor ref: fast_onionskin_create 152 | return os.urandom(TOR_DIGEST_LEN) 153 | 154 | def complete_handshake(self, handshake_response): 155 | # tor ref: fast_client_handshake 156 | peer_value = handshake_response[:TOR_DIGEST_LEN] 157 | key_hash = handshake_response[TOR_DIGEST_LEN:] 158 | shared_secret = self.handshake + peer_value 159 | computed_auth, key_material = kdf_tor(shared_secret) 160 | if computed_auth != key_hash: 161 | raise KeyAgreementError('Auth input does not match.') 162 | return key_material[:KeyAgreement.KEY_MATERIAL_LENGTH] 163 | 164 | 165 | class NtorKeyAgreement(KeyAgreement): 166 | TYPE = 2 167 | 168 | def __init__(self, onion_router): 169 | # 5.1.4. The "ntor" handshake 170 | 171 | # This handshake uses a set of DH handshakes to compute a set of 172 | # shared keys which the client knows are shared only with a particular 173 | # server, and the server knows are shared with whomever sent the 174 | # original handshake (or with nobody at all). Here we use the 175 | # "curve25519" group and representation as specified in "Curve25519: 176 | # new Diffie-Hellman speed records" by D. J. Bernstein. 177 | 178 | # [The ntor handshake was added in Tor 0.2.4.8-alpha.] 179 | 180 | # In this section, define: 181 | # H(x,t) as HMAC_SHA256 with message x and key t. 182 | # H_LENGTH = 32. 183 | # ID_LENGTH = 20. 184 | # G_LENGTH = 32 185 | # PROTOID = "ntor-curve25519-sha256-1" 186 | # t_mac = PROTOID | ":mac" 187 | # t_key = PROTOID | ":key_extract" 188 | # t_verify = PROTOID | ":verify" 189 | # MULT(a,b) = the multiplication of the curve25519 point 'a' by the 190 | # scalar 'b'. 191 | # G = The preferred base point for curve25519 ([9]) 192 | # KEYGEN() = The curve25519 key generation algorithm, returning 193 | # a private/public keypair. 194 | # m_expand = PROTOID | ":key_expand" 195 | 196 | # H is defined as hmac() 197 | # MULT is included in the curve25519 library as get_shared_key() 198 | # KEYGEN() is curve25519.Private() 199 | super().__init__(onion_router) 200 | 201 | self.protoid = b'ntor-curve25519-sha256-1' 202 | self.t_mac = self.protoid + b':mac' 203 | self.t_key = self.protoid + b':key_extract' 204 | self.t_verify = self.protoid + b':verify' 205 | self.m_expand = self.protoid + b':key_expand' 206 | 207 | # To perform the handshake, the client needs to know an identity key 208 | # digest for the server, and an ntor onion key (a curve25519 public 209 | # key) for that server. Call the ntor onion key "B". The client 210 | # generates a temporary keypair: 211 | # x,X = KEYGEN() 212 | self._x = curve25519_private() 213 | self._X = curve25519_public_from_private(self._x) 214 | 215 | self._onion_router = onion_router 216 | 217 | @property 218 | def _fingerprint_bytes(self): 219 | return self._onion_router.fingerprint 220 | 221 | @cached_property 222 | def _B(self): # noqa: N802 223 | return curve25519_public_from_bytes(self._onion_router.descriptor.ntor_key) # noqa: N802 224 | 225 | @cached_property 226 | def handshake(self): 227 | # and generates a client-side handshake with contents: 228 | # NODEID Server identity digest [ID_LENGTH bytes] 229 | # KEYID KEYID(B) [H_LENGTH bytes] 230 | # CLIENT_PK X [G_LENGTH bytes] 231 | handshake = self._fingerprint_bytes 232 | handshake += curve25519_to_bytes(self._B) 233 | handshake += curve25519_to_bytes(self._X) 234 | return handshake 235 | 236 | def complete_handshake(self, handshake_response): 237 | # The server's handshake reply is: 238 | # SERVER_PK Y [G_LENGTH bytes] 239 | # AUTH H(auth_input, t_mac) [H_LENGTH bytes] 240 | y = handshake_response[:32] # ntor data curve25519::public_key::key_size_in_bytes 241 | auth = handshake_response[32:] # ntor auth is SHA256, 32 in bytes 242 | assert len(auth) == 32 # 243 | 244 | # The client then checks Y is in G^* [see NOTE below], and computes 245 | 246 | # secret_input = EXP(Y,x) | EXP(B,x) | ID | B | X | Y | PROTOID 247 | si = curve25519_get_shared(self._x, curve25519_public_from_bytes(y)) 248 | si += curve25519_get_shared(self._x, self._B) 249 | si += self._fingerprint_bytes 250 | si += curve25519_to_bytes(self._B) 251 | si += curve25519_to_bytes(self._X) 252 | si += y 253 | si += b'ntor-curve25519-sha256-1' 254 | 255 | # KEY_SEED = H(secret_input, t_key) 256 | # verify = H(secret_input, t_verify) 257 | key_seed = hmac(self.t_key, si) 258 | verify = hmac(self.t_verify, si) 259 | 260 | # auth_input = verify | ID | B | Y | X | PROTOID | "Server" 261 | ai = verify 262 | ai += self._fingerprint_bytes 263 | ai += curve25519_to_bytes(self._B) 264 | ai += y 265 | ai += curve25519_to_bytes(self._X) 266 | ai += self.protoid 267 | ai += b'Server' 268 | 269 | # The client verifies that AUTH == H(auth_input, t_mac). 270 | if auth != hmac(self.t_mac, ai): 271 | raise KeyAgreementError('Auth input does not match.') 272 | 273 | # Both parties check that none of the EXP() operations produced the 274 | # point at infinity. [NOTE: This is an adequate replacement for 275 | # checking Y for group membership, if the group is curve25519.] 276 | 277 | # Both parties now have a shared value for KEY_SEED. They expand this 278 | # into the keys needed for the Tor relay protocol, using the KDF 279 | # described in 5.2.2 and the tag m_expand. 280 | 281 | # 5.2.2. KDF-RFC5869 282 | 283 | # For newer KDF needs, Tor uses the key derivation function HKDF from 284 | # RFC5869, instantiated with SHA256. (This is due to a construction 285 | # from Krawczyk.) The generated key material is: 286 | 287 | # K = K_1 | K_2 | K_3 | ... 288 | 289 | # Where H(x,t) is HMAC_SHA256 with value x and key t 290 | # and K_1 = H(m_expand | INT8(1) , KEY_SEED ) 291 | # and K_(i+1) = H(K_i | m_expand | INT8(i+1) , KEY_SEED ) 292 | # and m_expand is an arbitrarily chosen value, 293 | # and INT8(i) is a octet with the value "i". 294 | 295 | # In RFC5869's vocabulary, this is HKDF-SHA256 with info == m_expand, 296 | # salt == t_key, and IKM == secret_input. 297 | # WARN: length must be 92 298 | # 72 + byte_type rend_nonce [20]; << ignored now 299 | return hkdf_sha256(key_seed, length=KeyAgreement.KEY_MATERIAL_LENGTH, info=self.m_expand) 300 | -------------------------------------------------------------------------------- /torpy/parsers.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import re 17 | import logging 18 | 19 | from torpy.crypto_common import b64decode 20 | 21 | logger = logging.getLogger(__name__) 22 | 23 | 24 | class HSDescriptorParser: 25 | regex = re.compile( 26 | """\ 27 | introduction-points 28 | -----BEGIN MESSAGE----- 29 | (.+?) 30 | -----END MESSAGE-----""", 31 | flags=re.DOTALL | re.IGNORECASE, 32 | ) 33 | 34 | @staticmethod 35 | def parse(data): 36 | m = __class__.regex.search(data) 37 | if m: 38 | return m.group(1) 39 | else: 40 | logger.error("Can't parse HSDescriptor: %r", data) 41 | raise Exception("Can't parse HSDescriptor") 42 | 43 | 44 | class RouterDescriptorParser: 45 | regex = re.compile( 46 | r"""\ 47 | onion-key 48 | -----BEGIN RSA PUBLIC KEY----- 49 | (?P.+?) 50 | -----END RSA PUBLIC KEY----- 51 | signing-key 52 | -----BEGIN RSA PUBLIC KEY----- 53 | (?P.+?) 54 | -----END RSA PUBLIC KEY----- 55 | .+? 56 | ntor-onion-key (?P[^\n]+)""", 57 | flags=re.DOTALL | re.IGNORECASE, 58 | ) 59 | 60 | @staticmethod 61 | def parse(data): 62 | m = __class__.regex.search(data) 63 | if m: 64 | return {k: b64decode(v) for k, v in m.groupdict().items()} 65 | else: 66 | logger.debug("Can't parse router descriptor: %r", data) 67 | raise Exception("Can't parse router descriptor") 68 | 69 | 70 | class IntroPointParser: 71 | regex = re.compile( 72 | r"""\ 73 | introduction-point (?P[^\n]+) 74 | ip-address (?P[^\n]+) 75 | onion-port (?P[0-9]+) 76 | onion-key 77 | -----BEGIN RSA PUBLIC KEY----- 78 | (?P.+?) 79 | -----END RSA PUBLIC KEY----- 80 | service-key 81 | -----BEGIN RSA PUBLIC KEY----- 82 | (?P.+?) 83 | -----END RSA PUBLIC KEY-----""", 84 | flags=re.DOTALL | re.IGNORECASE, 85 | ) 86 | 87 | @staticmethod 88 | def _decode(d): 89 | for k in ('onion_key', 'service_key'): 90 | d[k] = b64decode(d[k]) 91 | return d 92 | 93 | @staticmethod 94 | def parse(data): 95 | res = [__class__._decode(m.groupdict()) for m in __class__.regex.finditer(data)] 96 | return res 97 | -------------------------------------------------------------------------------- /torpy/stream.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import time 17 | import socket 18 | import logging 19 | import threading 20 | from enum import unique, Enum, auto 21 | from selectors import EVENT_READ, DefaultSelector 22 | from contextlib import contextmanager 23 | 24 | from torpy.cells import ( 25 | CellRelayEnd, 26 | StreamReason, 27 | CellRelayData, 28 | CellRelayBegin, 29 | RelayedTorCell, 30 | CellRelaySendMe, 31 | CellRelayBeginDir, 32 | CellRelayConnected, 33 | ) 34 | from torpy.utils import chunks, hostname_key 35 | from torpy.hiddenservice import HiddenService 36 | 37 | logger = logging.getLogger(__name__) 38 | 39 | 40 | class TorWindow: 41 | def __init__(self, start=1000, increment=100): 42 | self._lock = threading.Lock() 43 | self._deliver = self._package = self._start = start 44 | self._increment = increment 45 | 46 | def need_sendme(self): 47 | with self._lock: 48 | if self._deliver > (self._start - self._increment): 49 | return False 50 | 51 | self._deliver += self._increment 52 | return True 53 | 54 | def deliver_dec(self): 55 | with self._lock: 56 | self._deliver -= 1 57 | 58 | def package_dec(self): 59 | with self._lock: 60 | self._package -= 1 61 | 62 | def package_inc(self): 63 | with self._lock: 64 | self._package += self._increment 65 | 66 | 67 | class TorSocketLoop(threading.Thread): 68 | def __init__(self, our_sock, send_func): 69 | super().__init__(name='SocketLoop{:x}'.format(our_sock.fileno())) 70 | self._our_sock = our_sock 71 | self._send_func = send_func 72 | self._do_loop = True 73 | 74 | self._cntrl_l = threading.Lock() 75 | self._cntrl_r, self._cntrl_w = socket.socketpair() 76 | 77 | self._selector = DefaultSelector() 78 | self._selector.register(self._our_sock, EVENT_READ, self._do_recv) 79 | self._selector.register(self._cntrl_r, EVENT_READ, self._do_stop) 80 | 81 | def _do_recv(self, sock): 82 | try: 83 | data = sock.recv(1024) 84 | self._send_func(data) 85 | except ConnectionResetError: 86 | logger.debug('Client was badly disconnected...') 87 | 88 | def _do_stop(self, sock): 89 | self._do_loop = False 90 | 91 | def _cleanup(self): 92 | with self._cntrl_l: 93 | logger.debug('Cleanup') 94 | self._selector.unregister(self._cntrl_r) 95 | self._cntrl_w.close() 96 | self._cntrl_r.close() 97 | self.close_sock() 98 | self._selector.close() 99 | 100 | @property 101 | def fileno(self): 102 | if not self._our_sock: 103 | return None 104 | return self._our_sock.fileno() 105 | 106 | def close_sock(self): 107 | if not self._our_sock: 108 | return 109 | self._selector.unregister(self._our_sock) 110 | self._our_sock.close() 111 | self._our_sock = None 112 | 113 | def stop(self): 114 | # Because stop could be called twice 115 | with self._cntrl_l: 116 | logger.debug('Stopping...') 117 | self._cntrl_w.send(b'\1') 118 | 119 | def run(self): 120 | logger.debug('Starting...') 121 | while self._do_loop: 122 | events = self._selector.select() 123 | for key, _ in events: 124 | callback = key.data 125 | callback(key.fileobj) 126 | 127 | self._cleanup() 128 | logger.debug('Stopped!') 129 | 130 | def append(self, data): 131 | self._our_sock.send(data) 132 | 133 | 134 | @unique 135 | class StreamState(Enum): 136 | Connecting = auto() 137 | Connected = auto() 138 | Disconnected = auto() 139 | Closed = auto() 140 | 141 | 142 | class TorStream: 143 | """This tor stream object implements socket-like interface.""" 144 | 145 | def __init__(self, id, circuit, auth_data=None): 146 | logger.info('Stream #%i: creating attached to #%x circuit...', id, circuit.id) 147 | self._id = id 148 | self._circuit = circuit 149 | self._auth_data = auth_data or {} 150 | 151 | self._buffer = bytearray() 152 | self._data_lock = threading.Lock() 153 | self._has_data = threading.Event() 154 | self._received_callbacks = [] 155 | 156 | self._conn_timeout = 30 157 | self._recv_timeout = 60 158 | 159 | self._state = StreamState.Closed 160 | self._close_lock = threading.Lock() 161 | 162 | self._window = TorWindow(start=500, increment=50) 163 | 164 | self._loop = None 165 | 166 | def __enter__(self): 167 | """Start using the stream.""" 168 | return self 169 | 170 | def __exit__(self, exc_type, exc_val, exc_tb): 171 | """Close the stream.""" 172 | self.close() 173 | 174 | @property 175 | def id(self): 176 | return self._id 177 | 178 | @property 179 | def has_socket_loop(self): 180 | return self._loop is not None 181 | 182 | @property 183 | def state(self): 184 | return self._state 185 | 186 | def handle_cell(self, cell): 187 | logger.debug(cell) 188 | if isinstance(cell, CellRelayConnected): 189 | self._connected(cell) 190 | elif isinstance(cell, CellRelayEnd): 191 | self._end(cell) 192 | self._call_received() 193 | elif isinstance(cell, CellRelayData): 194 | self._append(cell.data) 195 | self._window.deliver_dec() 196 | if self._window.need_sendme(): 197 | self.send_relay(CellRelaySendMe(circuit_id=cell.circuit_id)) 198 | self._call_received() 199 | elif isinstance(cell, CellRelaySendMe): 200 | logger.debug('Stream #%i: sendme received', self.id) 201 | self._window.package_inc() 202 | else: 203 | raise Exception('Unknown stream cell received: %r', type(cell)) 204 | 205 | def register(self, callback): 206 | self._received_callbacks.append(callback) 207 | 208 | def unregister(self, callback): 209 | self._received_callbacks.remove(callback) 210 | 211 | def _call_received(self): 212 | for callback in self._received_callbacks: 213 | callback(self, EVENT_READ) 214 | 215 | def send_relay(self, inner_cell): 216 | return self._circuit.send_relay(inner_cell, stream_id=self.id) 217 | 218 | def _append(self, data): 219 | with self._close_lock, self._data_lock: 220 | if self._state == StreamState.Closed: 221 | logger.warning('Stream #%i: closed (but received %r)', self.id, data) 222 | return 223 | 224 | if self.has_socket_loop: 225 | logger.debug('Stream #%i: append %i (to sock #%r)', self.id, len(data), self._loop.fileno) 226 | self._loop.append(data) 227 | else: 228 | logger.debug('Stream #%i: append %i (to buffer)', self.id, len(data)) 229 | self._buffer.extend(data) 230 | self._has_data.set() 231 | 232 | def close(self): 233 | logger.info('Stream #%i: closing (state = %s)...', self.id, self._state.name) 234 | 235 | with self._close_lock: 236 | if self._state == StreamState.Closed: 237 | logger.warning('Stream #%i: closed already', self.id) 238 | return 239 | 240 | if self._state == StreamState.Connected: 241 | self.send_end() 242 | 243 | if self.has_socket_loop: 244 | self.close_socket() 245 | 246 | self._circuit.remove_stream(self) 247 | 248 | self._state = StreamState.Closed 249 | logger.debug('Stream #%i: closed', self.id) 250 | 251 | def _prepare_address(self, address): 252 | if isinstance(address[0], HiddenService): 253 | return address[0], (address[0].onion, address[1]) 254 | elif address[0].endswith('.onion'): 255 | host_key = hostname_key(address[0]) 256 | descriptor_cookie, auth_type = self._auth_data.get(host_key, HiddenService.HS_NO_AUTH) 257 | return HiddenService(address[0], descriptor_cookie, auth_type), address 258 | else: 259 | return None, address 260 | 261 | def connect(self, address): 262 | logger.info('Stream #%i: connecting to %r', self.id, address) 263 | assert self._state == StreamState.Closed 264 | self._state = StreamState.Connecting 265 | 266 | hidden_service, address = self._prepare_address(address) 267 | if hidden_service: 268 | self._circuit.extend_to_hidden(hidden_service) 269 | 270 | # Now we can connect to its address 271 | self._connect(address) 272 | 273 | def _wait_connected(self, address, timeout): 274 | start_time = time.time() 275 | while True: 276 | if time.time() - start_time > timeout: 277 | raise TimeoutError('Could not connect to %r' % (address,)) 278 | 279 | if self._state == StreamState.Connected: 280 | return 281 | elif self._state == StreamState.Closed: 282 | raise ConnectionError('Could not connect to %r' % (address,)) 283 | 284 | time.sleep(0.2) 285 | 286 | def connect_dir(self): 287 | logger.info('Stream #%i: connecting to hsdir', self.id) 288 | assert self._state == StreamState.Closed 289 | self._state = StreamState.Connecting 290 | 291 | inner_cell = CellRelayBeginDir() 292 | self.send_relay(inner_cell) 293 | self._wait_connected('hsdir', self._conn_timeout) 294 | 295 | def _connect(self, address): 296 | inner_cell = CellRelayBegin(address[0], address[1]) 297 | self.send_relay(inner_cell) 298 | self._wait_connected(address, self._conn_timeout) 299 | 300 | def _connected(self, cell_connected): 301 | self._state = StreamState.Connected 302 | logger.info('Stream #%i: connected (remote ip %r)', self.id, cell_connected.address) 303 | 304 | def send(self, data): 305 | for chunk in chunks(data, RelayedTorCell.MAX_PAYLOD_SIZE): 306 | self._circuit.last_node.window.package_dec() 307 | self.send_relay(CellRelayData(chunk, self._circuit.id)) 308 | 309 | def send_end(self) -> None: 310 | self.send_relay(CellRelayEnd(StreamReason.DONE, self._circuit.id)) 311 | 312 | def send_sendme(self): 313 | self.send_relay(CellRelaySendMe(circuit_id=self._circuit.id)) 314 | 315 | def _end(self, cell_end): 316 | logger.info('Stream #%i: remote disconnected (reason = %s)', self.id, cell_end.reason.name) 317 | with self._close_lock, self._data_lock: 318 | # For case when _end arrived later than we close 319 | if self._state == StreamState.Connected: 320 | self._state = StreamState.Disconnected 321 | 322 | if self.has_socket_loop: 323 | logger.debug('Close our sock...') 324 | self._loop.close_sock() 325 | else: 326 | self._has_data.set() 327 | 328 | def _create_socket_loop(self): 329 | our_sock, client_sock = socket.socketpair() 330 | logger.debug('Created sock pair: our_sock = %x, client_sock = %x', our_sock.fileno(), client_sock.fileno()) 331 | 332 | # Flush data 333 | with self._data_lock: 334 | if len(self._buffer): 335 | logger.debug('Flush buffer') 336 | our_sock.send(self._buffer) 337 | self._buffer.clear() 338 | 339 | # Create proxy loop 340 | return client_sock, TorSocketLoop(our_sock, self.send) 341 | 342 | @contextmanager 343 | def as_socket(self): 344 | logger.debug('[as_socket] start') 345 | client_socket = self.create_socket() 346 | try: 347 | yield client_socket 348 | finally: 349 | logger.debug('[as_socket] finally') 350 | client_socket.close() 351 | self.close_socket() 352 | 353 | def create_socket(self, timeout=30): 354 | client_sock, self._loop = self._create_socket_loop() 355 | if timeout: 356 | client_sock.settimeout(timeout) 357 | self._loop.start() 358 | 359 | return client_sock 360 | 361 | def close_socket(self): 362 | self._loop.stop() 363 | self._loop.join() 364 | 365 | def recv(self, bufsize): 366 | if self.has_socket_loop: 367 | raise Exception('You must use socket') 368 | 369 | if self._state == StreamState.Closed: 370 | raise Exception("You can't recv closed stream") 371 | 372 | signaled = self._has_data.wait(self._recv_timeout) 373 | if not signaled: 374 | raise Exception('recv timeout') 375 | 376 | # If remote side already send 'end cell' but we still 377 | # has some data - we keep receiving 378 | if self._state == StreamState.Disconnected and not self._buffer: 379 | return b'' 380 | 381 | with self._data_lock: 382 | if bufsize == -1: 383 | to_read = len(self._buffer) 384 | else: 385 | to_read = min(len(self._buffer), bufsize) 386 | result = self._buffer[:to_read] 387 | self._buffer = self._buffer[to_read:] 388 | logger.debug('Stream #%i: read %i (left %i)', self.id, to_read, len(self._buffer)) 389 | 390 | # Clear 'has_data' flag only if we don't have more data and not disconnected 391 | if not self._buffer and self._state != StreamState.Disconnected: 392 | self._has_data.clear() 393 | 394 | return result 395 | 396 | 397 | class StreamsList: 398 | LOCK = threading.Lock() 399 | GLOBAL_STREAM_ID = 0 400 | 401 | def __init__(self, circuit, auth_data): 402 | self._stream_map = {} 403 | self._circuit = circuit 404 | self._auth_data = auth_data 405 | 406 | @staticmethod 407 | def get_next_stream_id(): 408 | with StreamsList.LOCK: 409 | StreamsList.GLOBAL_STREAM_ID += 1 410 | return StreamsList.GLOBAL_STREAM_ID 411 | 412 | def create_new(self): 413 | stream = TorStream(self.get_next_stream_id(), self._circuit, self._auth_data) 414 | self._stream_map[stream.id] = stream 415 | return stream 416 | 417 | def values(self): 418 | return self._stream_map.values() 419 | 420 | def remove(self, tor_stream): 421 | stream = self._stream_map.pop(tor_stream.id, None) 422 | if not stream: 423 | logger.debug('Stream #%i: not found in stream map', tor_stream.id) 424 | 425 | def get_by_id(self, stream_id): 426 | return self._stream_map.get(stream_id, None) 427 | -------------------------------------------------------------------------------- /torpy/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import os 17 | import sys 18 | import gzip 19 | import zlib 20 | import time 21 | import logging 22 | import threading 23 | import contextlib 24 | from base64 import b64encode 25 | from urllib import request 26 | 27 | logger = logging.getLogger(__name__) 28 | 29 | 30 | def register_logger(verbose, log_file=None): 31 | fmt = '[%(asctime)s] [%(threadName)-16s] %(message)s' if verbose else '%(message)s' 32 | lvl = logging.DEBUG if verbose else logging.INFO 33 | if not verbose: 34 | logging.getLogger('requests').setLevel(logging.CRITICAL) 35 | handlers = [logging.StreamHandler()] 36 | if log_file: 37 | handlers.append(logging.FileHandler(log_file)) 38 | logging.basicConfig(format=fmt, level=lvl, handlers=handlers) 39 | 40 | 41 | def to_hex(b): 42 | return ' '.join('{:02x}'.format(x) for x in b) 43 | 44 | 45 | def fp_to_str(fp): 46 | return b64encode(fp).decode() 47 | 48 | 49 | class cached_property: # noqa: N801 50 | def __init__(self, func): 51 | self.__doc__ = func.__doc__ 52 | self.func = func 53 | self.lock = threading.RLock() 54 | 55 | def __get__(self, obj, cls): 56 | """Check whether return value already exists and return it.""" 57 | if obj is None: 58 | return self 59 | 60 | with self.lock: 61 | value = obj.__dict__[self.func.__name__] = self.func(obj) 62 | return value 63 | 64 | 65 | def log_retry(exc_info, msg, no_traceback=None): 66 | if no_traceback is not None and exc_info[0] not in no_traceback: 67 | logging.error('[ignored]', exc_info=exc_info[1]) 68 | else: 69 | logger.error('[ignored] %s.%s: %s', exc_info[0].__module__, exc_info[0].__qualname__, str(exc_info[1])) 70 | logger.warning(msg) 71 | 72 | 73 | def retry(times, exceptions, delay=1, backoff=0, log_func=None): 74 | def decorator(func): 75 | def newfn(*args, **kwargs): 76 | left = times 77 | while left: 78 | try: 79 | return func(*args, **kwargs) 80 | except exceptions: 81 | if log_func: 82 | exc_info = sys.exc_info() 83 | try: 84 | log_func(exc_info) 85 | finally: 86 | del exc_info 87 | else: 88 | logger.info( 89 | 'Exception thrown when attempting to run %s, attempt %d of %d', 90 | func, 91 | times - left, 92 | times, 93 | exc_info=True, 94 | ) 95 | left -= 1 96 | if not left: 97 | raise 98 | if delay: 99 | total_delay = delay + (times - left) * backoff 100 | logger.info('Wait %i sec before next retry', total_delay) 101 | time.sleep(total_delay) 102 | 103 | return newfn 104 | 105 | return decorator 106 | 107 | 108 | @contextlib.contextmanager 109 | def ignore(comment, exceptions=None, log_func=None): 110 | exceptions = exceptions or (Exception,) 111 | try: 112 | yield 113 | except exceptions: 114 | if log_func: 115 | exc_info = sys.exc_info() 116 | log_func(exc_info, comment) 117 | del exc_info 118 | else: 119 | logger.info(comment, exc_info=True) 120 | 121 | 122 | def scheme_to_port(scheme): 123 | if scheme == 'http': 124 | return 80 125 | elif scheme == 'https': 126 | return 443 127 | elif scheme == 'ftp': 128 | return 21 129 | 130 | 131 | def chunks(lst, n): 132 | """Yield successive n-sized chunks from l.""" 133 | for i in range(0, len(lst), n): 134 | yield lst[i:i + n] 135 | 136 | 137 | def recv_exact(sock, n): 138 | data = b'' 139 | while n: 140 | chunk = sock.recv(n) 141 | if not chunk: 142 | break 143 | n -= len(chunk) 144 | data += chunk 145 | return data 146 | 147 | 148 | def coro_recv_exact(n): 149 | data = b'' 150 | while n: 151 | chunk = yield n 152 | if not chunk: 153 | break 154 | n -= len(chunk) 155 | data += chunk 156 | return data 157 | 158 | 159 | def recv_all(sock): 160 | """Receive data until connection is closed.""" 161 | data = b'' 162 | while True: 163 | chunk = sock.recv(1024) 164 | if not chunk: 165 | break 166 | data += chunk 167 | return data 168 | 169 | 170 | class AuthType: 171 | No = 0 172 | Basic = 1 173 | Stealth = 2 174 | 175 | 176 | def user_data_dir(app_name): 177 | """Return full path to the user-specific data dir for this application.""" 178 | if sys.platform == 'win32': 179 | app_name = os.path.join(app_name, app_name) # app_author + app_name 180 | path = os.path.expandvars(r'%APPDATA%') 181 | elif sys.platform == 'darwin': 182 | path = os.path.expanduser('~/Library/Application Support/') 183 | else: 184 | path = os.getenv('XDG_DATA_HOME', os.path.expanduser('~/.local/share')) 185 | return os.path.join(path, app_name) 186 | 187 | 188 | def http_get(url, timeout=10, headers=None): 189 | opener = request.build_opener() 190 | 191 | real_headers = {'Accept-encoding': 'gzip, deflate'} 192 | real_headers.update(headers or {}) 193 | opener.addheaders = [(k, v) for k, v in real_headers.items()] 194 | 195 | with opener.open(url, timeout=timeout) as response: 196 | data = response.read() 197 | if response.info().get('Content-Encoding') == 'gzip': 198 | data = gzip.decompress(data) 199 | elif response.info().get('Content-Encoding') == 'deflate': 200 | data = zlib.decompress(data) 201 | return data.decode('utf-8') 202 | 203 | 204 | def hostname_key(hostname): 205 | return '.'.join(hostname.split('.')[-2:]) 206 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | {py36,py37,py38,py39}-unit 4 | {py36,py37,py38,py39}-integration 5 | flake8 6 | 7 | [flake8] 8 | max-line-length = 120 9 | ignore = D100, D101, D102, D103, D104, D107 10 | import-order-style = pep8 11 | application_import_names = torpy 12 | min_python_version = 3.6.0 13 | inline-quotes = ' 14 | 15 | [testenv] 16 | changedir = tests 17 | deps = -rrequirements-test.txt 18 | extras = requests 19 | commands = 20 | unit: py.test --cov torpy unittest 21 | integration: py.test -s --cov torpy integration 22 | 23 | 24 | [testenv:flake8] 25 | basepython = python3 26 | deps = -rrequirements-flake8.txt 27 | commands = 28 | flake8 ../torpy/ ../tests/ ../setup.py --------------------------------------------------------------------------------