├── AUTHORS ├── tests ├── __init__.py ├── unittest │ ├── __init__.py │ ├── test_hiddenservice.py │ ├── data │ │ ├── dir_certs │ │ │ └── dir_cert_real │ │ └── network_status │ │ │ ├── consensus │ │ │ ├── consensus_extra │ │ │ ├── consensus.json │ │ │ └── consensus_extra.json │ ├── test_consensus.py │ ├── test_crypto.py │ └── test_documents.py └── integration │ ├── __init__.py │ └── test_integration.py ├── torpy ├── cli │ ├── __init__.py │ ├── console.py │ └── socks.py ├── http │ ├── __init__.py │ ├── base.py │ ├── client.py │ ├── requests.py │ ├── urlopener.py │ └── adapter.py ├── __init__.py ├── __main__.py ├── documents │ ├── __init__.py │ ├── factory.py │ ├── dir_key_certificate.py │ ├── network_status_diff.py │ ├── items.py │ └── basics.py ├── crypto.py ├── cache_storage.py ├── client.py ├── parsers.py ├── crypto_state.py ├── crypto_common.py ├── guard.py ├── utils.py ├── cell_socket.py ├── keyagreement.py ├── hiddenservice.py └── stream.py ├── MANIFEST.in ├── setup.cfg ├── requirements-test.txt ├── pytest.ini ├── .github └── FUNDING.yml ├── requirements-flake8.txt ├── requirements.txt ├── .travis.yml ├── appveyor.yml ├── tox.ini ├── .gitignore ├── setup.py ├── README.md └── LICENSE /AUTHORS: -------------------------------------------------------------------------------- 1 | James Brown -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /torpy/cli/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /torpy/http/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/unittest/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/integration/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.md 2 | include LICENSE -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | license_files = LICENSE.txt -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | coverage 3 | pytest-cov -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | log_cli=true 3 | log_level=NOTSET 4 | log_format=[%(asctime)s] [%(threadName)-16s] %(message)s -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | custom: ['https://btc.com/16mF9TYaJKkb9eGbZ5jGuJbodTF3mYvcRF', 'https://coinrequest.io/request/anscJbCsqucdJ1m'] 4 | -------------------------------------------------------------------------------- /requirements-flake8.txt: -------------------------------------------------------------------------------- 1 | flake8 2 | flake8-colors 3 | flake8-quotes 4 | flake8-docstrings>=0.2.7 5 | pydocstyle<4 6 | flake8-import-order>=0.9 7 | flake8-typing-imports>=1.1 8 | pep8-naming 9 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile setup.py 6 | # 7 | cffi==1.14.4 # via cryptography 8 | cryptography==3.4.4 9 | pycparser==2.20 # via cffi 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | os: linux 2 | dist: bionic 3 | language: python 4 | python: 5 | - "3.6" 6 | - "3.7" 7 | - "3.8" 8 | - "3.9" 9 | install: 10 | - pip install tox-travis coveralls 11 | script: 12 | - tox -v 13 | after_success: 14 | - cd tests && coveralls 15 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | --- 2 | environment: 3 | matrix: 4 | - TOXENV: py36-unit 5 | - TOXENV: py36-integration 6 | - TOXENV: py37-unit 7 | - TOXENV: py37-integration 8 | 9 | build: off 10 | 11 | install: 12 | - pip install tox 13 | 14 | test_script: 15 | - tox -v 16 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | {py36,py37,py38,py39}-unit 4 | {py36,py37,py38,py39}-integration 5 | flake8 6 | 7 | [flake8] 8 | max-line-length = 120 9 | ignore = D100, D101, D102, D103, D104, D107 10 | import-order-style = pep8 11 | application_import_names = torpy 12 | min_python_version = 3.6.0 13 | inline-quotes = ' 14 | 15 | [testenv] 16 | changedir = tests 17 | deps = -rrequirements-test.txt 18 | extras = requests 19 | commands = 20 | unit: py.test --cov torpy unittest 21 | integration: py.test -s --cov torpy integration 22 | 23 | 24 | [testenv:flake8] 25 | basepython = python3 26 | deps = -rrequirements-flake8.txt 27 | commands = 28 | flake8 ../torpy/ ../tests/ ../setup.py -------------------------------------------------------------------------------- /torpy/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | from torpy.client import TorClient 17 | 18 | __all__ = ['TorClient'] 19 | -------------------------------------------------------------------------------- /torpy/__main__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | from torpy.cli.console import main 17 | 18 | if __name__ == '__main__': 19 | main() 20 | -------------------------------------------------------------------------------- /torpy/documents/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | from torpy.documents.basics import TorDocument 17 | from torpy.documents.factory import TorDocumentsFactory 18 | 19 | 20 | __all__ = ['TorDocument', 'TorDocumentsFactory'] 21 | -------------------------------------------------------------------------------- /torpy/documents/factory.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | from torpy.documents import TorDocument 17 | 18 | 19 | class TorDocumentsFactory: 20 | @staticmethod 21 | def parse(raw_string, kwargs=None, possible=None): 22 | kwargs = kwargs or {} 23 | possible = possible or TorDocument.__subclasses__() 24 | 25 | for doc_cls in possible: 26 | if doc_cls.check_start(raw_string): 27 | return doc_cls(raw_string, **kwargs) 28 | 29 | return None 30 | -------------------------------------------------------------------------------- /tests/unittest/test_hiddenservice.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from torpy.hiddenservice import HiddenService 4 | 5 | 6 | @pytest.mark.parametrize( 7 | 'onion_hostname, expected_result', 8 | [ 9 | ('facebookcorewwwi.onion', 10 | ('facebookcorewwwi', b'(\x04@\xb9\xca\x13\xa2KZ\xc8', None)), 11 | ('sss.subdomain.facebookcorewwwi.onion', 12 | ('facebookcorewwwi', b'(\x04@\xb9\xca\x13\xa2KZ\xc8', None)), 13 | ('5gdvpfoh6kb2iqbizb37lzk2ddzrwa47m6rpdueg2m656fovmbhoptqd.onion', 14 | ('5gdvpfoh6kb2iqbizb37lzk2ddzrwa47m6rpdueg2m656fovmbhoptqd', None, 15 | b'\xe9\x87W\x95\xc7\xf2\x83\xa4@(\xc8w\xf5\xe5Z\x18\xf3\x1b\x03\x9fg\xa2\xf1\xd0\x86\xd3=\xdf\x15\xd5`N')), 16 | ('subd.5gdvpfoh6kb2iqbizb37lzk2ddzrwa47m6rpdueg2m656fovmbhoptqd.onion', 17 | ('5gdvpfoh6kb2iqbizb37lzk2ddzrwa47m6rpdueg2m656fovmbhoptqd', None, 18 | b'\xe9\x87W\x95\xc7\xf2\x83\xa4@(\xc8w\xf5\xe5Z\x18\xf3\x1b\x03\x9fg\xa2\xf1\xd0\x86\xd3=\xdf\x15\xd5`N')), 19 | ] 20 | ) 21 | def test_parse_onion(onion_hostname, expected_result): 22 | result = HiddenService.parse_onion(onion_hostname) 23 | assert result == expected_result 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv*/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # Idea project 107 | .idea/ 108 | bak/ 109 | .isort.cfg 110 | -------------------------------------------------------------------------------- /torpy/documents/dir_key_certificate.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import logging 17 | 18 | from torpy.documents.basics import TorDocument, TorDocumentObject 19 | from torpy.documents.items import Item, ItemDate, ItemInt, ItemMulti, ItemObject 20 | 21 | logger = logging.getLogger(__name__) 22 | 23 | 24 | class DirKeyCertificateObject(TorDocumentObject): 25 | 26 | START_ITEM = ItemInt('dir-key-certificate-version') 27 | 28 | ITEMS = [ 29 | # fingerprint 14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 30 | Item('fingerprint'), 31 | # dir-key-published 2019-06-01 00:00:00 32 | ItemDate('dir-key-published'), 33 | # dir-key-expires 2019-11-01 00:00:00 34 | ItemDate('dir-key-expires'), 35 | ItemMulti('dir-identity-key', 'rsa public key'), 36 | ItemMulti('dir-signing-key', 'rsa public key'), 37 | ItemMulti('dir-key-crosscert', 'id signature'), 38 | ItemMulti('dir-key-certification', 'signature'), 39 | ] 40 | 41 | 42 | class DirKeyCertificate(TorDocument, DirKeyCertificateObject): 43 | DOCUMENT_NAME = 'dir_key_certificate' 44 | 45 | 46 | class DirKeyCertificateList(TorDocument): 47 | DOCUMENT_NAME = 'dir_key_certificates' 48 | 49 | START_ITEM = '' 50 | 51 | ITEMS = [ItemObject(DirKeyCertificateObject, out_name='certs')] 52 | 53 | def find(self, identity): 54 | return next((cert for cert in self.certs if cert.fingerprint == identity), None) 55 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | from io import open 17 | from os import path 18 | 19 | from setuptools import setup, find_packages 20 | 21 | here = path.abspath(path.dirname(__file__)) 22 | 23 | with open(path.join(here, 'README.md'), encoding='utf-8') as f: 24 | long_description = f.read() 25 | 26 | setup( 27 | name='torpy', 28 | version='1.1.6', 29 | description='Pure python tor protocol implementation', 30 | long_description=long_description, 31 | long_description_content_type='text/markdown', 32 | url='https://github.com/torpyorg/torpy', 33 | author='James Brown', 34 | classifiers=[ 35 | 'Development Status :: 4 - Beta', 36 | 'Intended Audience :: Developers', 37 | 'License :: OSI Approved :: Apache Software License', 38 | 'Programming Language :: Python :: 3.6', 39 | 'Programming Language :: Python :: 3.7', 40 | 'Programming Language :: Python :: 3.8', 41 | 'Programming Language :: Python :: 3.9', 42 | ], 43 | keywords='python proxy anonymity privacy socks tor protocol onion hiddenservice', 44 | packages=find_packages(exclude=['tests']), 45 | python_requires='>=3.6', 46 | install_requires=['cryptography>=3.2'], 47 | extras_require={'requests': 'requests>2.9,!=2.17.0,!=2.18.0'}, 48 | entry_points={'console_scripts': ['torpy_cli=torpy.cli.console:main', 49 | 'torpy_socks=torpy.cli.socks:main'] 50 | }, 51 | project_urls={ 52 | 'Bug Reports': 'https://github.com/torpyorg/torpy/issues', 53 | 'Source': 'https://github.com/torpyorg/torpy/', 54 | }, 55 | ) 56 | -------------------------------------------------------------------------------- /torpy/http/base.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import logging 3 | 4 | from torpy.utils import hostname_key 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | class SocketProxy: 10 | def __init__(self, sock, tor_stream): 11 | self._sock = sock 12 | self._tor_stream = tor_stream 13 | 14 | def __getattr__(self, attr): 15 | """Proxying methods to real socket.""" 16 | if attr in self.__dict__: 17 | return getattr(self, attr) 18 | return getattr(self._sock, attr) 19 | 20 | @classmethod 21 | def rewrap(cls, prev_proxy, new_sock): 22 | return cls(new_sock, prev_proxy.tor_stream) 23 | 24 | @property 25 | def wrapped_sock(self): 26 | return self._sock 27 | 28 | @property 29 | def tor_stream(self): 30 | return self._tor_stream 31 | 32 | def close(self): 33 | logger.debug('[SocketProxy] close') 34 | self.close_tor_stream() 35 | self._sock.close() 36 | 37 | def close_tor_stream(self): 38 | self._tor_stream.close() 39 | 40 | 41 | class TorInfo: 42 | def __init__(self, guard, hops_count): 43 | self._guard = guard 44 | self._hops_count = hops_count 45 | self._circuits = {} 46 | self._lock = threading.Lock() 47 | 48 | def get_circuit(self, hostname): 49 | host_key = hostname_key(hostname) 50 | logger.debug('[TorInfo] Waiting lock...') 51 | with self._lock: 52 | logger.debug('[TorInfo] Got lock...') 53 | circuit = self._circuits.get(host_key) 54 | if not circuit: 55 | logger.debug('[TorInfo] Create new circuit for %s (key %s)', hostname, host_key) 56 | circuit = self._guard.create_circuit(self._hops_count) 57 | self._circuits[host_key] = circuit 58 | else: 59 | logger.debug('[TorInfo] Use existing...') 60 | return circuit 61 | 62 | def connect(self, address, timeout=30, source_address=None): 63 | circuit = self.get_circuit(address[0]) 64 | tor_stream = circuit.create_stream(address) 65 | logger.debug('[TorHTTPConnection] tor_stream create_socket') 66 | return SocketProxy(tor_stream.create_socket(), tor_stream) 67 | -------------------------------------------------------------------------------- /torpy/crypto.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import os 17 | import logging 18 | 19 | from torpy.crypto_common import sha1, aes_update, rsa_encrypt, rsa_load_der, aes_ctr_encryptor 20 | 21 | logger = logging.getLogger(__name__) 22 | 23 | 24 | TOR_DIGEST_LEN = 20 25 | 26 | 27 | def tor_digest(msg): 28 | return sha1(msg) 29 | 30 | 31 | def kdf_tor(shared_secret): 32 | # tor ref: crypto_expand_key_material_TAP 33 | t = shared_secret + bytes([0]) 34 | computed_auth = tor_digest(t) 35 | key_material = b'' 36 | for i in range(1, 5): 37 | t = shared_secret + bytes([i]) 38 | tsh = tor_digest(t) 39 | key_material += tsh 40 | return computed_auth, key_material 41 | 42 | 43 | # tor-spec.txt 0.3. 44 | KEY_LEN = 16 45 | PK_ENC_LEN = 128 46 | PK_PAD_LEN = 42 47 | 48 | PK_DATA_LEN = PK_ENC_LEN - PK_PAD_LEN 49 | PK_DATA_LEN_WITH_KEY = PK_DATA_LEN - KEY_LEN 50 | 51 | 52 | def hybrid_encrypt(data, rsa_key_der): 53 | """ 54 | Hybrid encryption scheme. 55 | 56 | Encrypt the entire contents of the byte array "data" with the given "TorPublicKey" according to 57 | the "hybrid encryption" scheme described in the main Tor specification (tor-spec.txt). 58 | """ 59 | rsa_key = rsa_load_der(rsa_key_der) 60 | 61 | if len(data) < PK_DATA_LEN: 62 | return rsa_encrypt(rsa_key, data) 63 | 64 | aes_key_bytes = os.urandom(KEY_LEN) 65 | 66 | # RSA(K | M1) --> C1 67 | m1 = data[:PK_DATA_LEN_WITH_KEY] 68 | c1 = rsa_encrypt(rsa_key, aes_key_bytes + m1) 69 | 70 | # AES_CTR(M2) --> C2 71 | m2 = data[PK_DATA_LEN_WITH_KEY:] 72 | aes_key = aes_ctr_encryptor(aes_key_bytes) 73 | c2 = aes_update(aes_key, m2) 74 | 75 | return c1 + c2 76 | -------------------------------------------------------------------------------- /torpy/http/client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import gzip 17 | import zlib 18 | import logging 19 | from io import BytesIO 20 | from http.client import parse_headers 21 | 22 | from torpy.utils import recv_all 23 | 24 | logger = logging.getLogger(__name__) 25 | 26 | 27 | class HttpStreamClient: 28 | def __init__(self, stream, host=None): 29 | self._stream = stream 30 | self._host = host 31 | 32 | def get(self, path, host=None, headers: dict = None): 33 | headers = headers or {} 34 | host = host or self._host 35 | if host: 36 | headers['Host'] = host 37 | headers_str = '\r\n'.join(f'{key}: {val}' for (key, val) in headers.items()) 38 | http_query = f'GET {path} HTTP/1.0\r\n{headers_str}\r\n\r\n' 39 | self._stream.send(http_query.encode()) 40 | 41 | raw_response = recv_all(self._stream) 42 | header, body = raw_response.split(b'\r\n\r\n', 1) 43 | 44 | f = BytesIO(header) 45 | request_line = f.readline().split(b' ') 46 | protocol, status = request_line[:2] 47 | status = int(status) 48 | 49 | headers = parse_headers(f) 50 | if headers['Content-Encoding'] == 'deflate': 51 | body = zlib.decompress(body) 52 | elif headers['Content-Encoding'] == 'gzip': 53 | body = gzip.decompress(body) 54 | 55 | if status != 200: 56 | logger.debug('raw_response = %s', raw_response) 57 | 58 | return status, body 59 | 60 | def close(self): 61 | self._stream.close() 62 | 63 | def __enter__(self): 64 | """Start using the http client.""" 65 | return self 66 | 67 | def __exit__(self, exc_type, exc_val, exc_tb): 68 | """Close the http client.""" 69 | self.close() 70 | -------------------------------------------------------------------------------- /torpy/cache_storage.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import os 17 | import logging 18 | 19 | from torpy.utils import user_data_dir 20 | from torpy.documents import TorDocument 21 | 22 | logger = logging.getLogger(__name__) 23 | 24 | 25 | class TorCacheStorage: 26 | def load(self, key): 27 | raise NotImplementedError() 28 | 29 | def load_document(self, doc_cls, **kwargs): 30 | assert issubclass(doc_cls, TorDocument) 31 | ident, content = self.load(doc_cls.DOCUMENT_NAME) 32 | if content: 33 | logger.info('Loading cached %s from %s: %s', doc_cls.__name__, self.__class__.__name__, ident) 34 | return doc_cls(content, **kwargs) 35 | else: 36 | return None 37 | 38 | def save(self, key, content): 39 | raise NotImplementedError() 40 | 41 | def save_document(self, doc): 42 | assert isinstance(doc, TorDocument) 43 | self.save(doc.DOCUMENT_NAME, doc.raw_string) 44 | 45 | 46 | class TorCacheDirStorage(TorCacheStorage): 47 | def __init__(self, base_dir=None): 48 | self._base_dir = base_dir or user_data_dir('torpy') 49 | if not os.path.isdir(self._base_dir): 50 | os.makedirs(self._base_dir) 51 | 52 | def load(self, key): 53 | file_path = os.path.join(self._base_dir, key) 54 | if os.path.isfile(file_path): 55 | with open(os.path.join(self._base_dir, key), 'r') as f: 56 | return file_path, f.read() 57 | else: 58 | return file_path, None 59 | 60 | def save(self, key, content): 61 | with open(os.path.join(self._base_dir, key), 'w') as f: 62 | f.write(content) 63 | 64 | 65 | class NoCacheStorage(TorCacheStorage): 66 | def load(self, key): 67 | return None, None 68 | 69 | def save(self, key, content): 70 | pass 71 | -------------------------------------------------------------------------------- /tests/unittest/data/dir_certs/dir_cert_real: -------------------------------------------------------------------------------- 1 | dir-key-certificate-version 3 2 | fingerprint 14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 3 | dir-key-published 2019-06-01 00:00:00 4 | dir-key-expires 2019-11-01 00:00:00 5 | dir-identity-key 6 | -----BEGIN RSA PUBLIC KEY----- 7 | MIIBigKCAYEA7cZXvDRxfjDYtr9/9UsQ852+6cmHMr8VVh8GkLwbq3RzqjkULwQ2 8 | R9mFvG4FnqMcMKXi62rYYA3fZL1afhT804cpvyp/D3dPM8QxW88fafFAgIFP4LiD 9 | 0JYjnF8cva5qZ0nzlWnMXLb32IXSvsGSE2FRyAV0YN9a6k967LSgCfUnZ+IKMezW 10 | 1vhL9YK4QIfsDowgtVsavg63GzGmA7JvZmn77+/J5wKz11vGr7Wttf8XABbH2taX 11 | O9j/KGBOX2OKhoF3mXfZSmUO2dV9NMwtkJ7zD///Ny6sfApWV6kVP4O9TdG3bAsl 12 | +fHCoCKgF/jAAWzh6VckQTOPzQZaH5aMWfXrDlzFWg17MjonI+bBTD2Ex2pHczzJ 13 | bN7coDMRH2SuOXv8wFf27KdUxZ/GcrXSRGzlRLygxqlripUanjVGN2JvrVQVr0kz 14 | pjNjiZl2z8ZyZ5d4zQuBi074JPGgx62xAstP37v1mPw14sIWfLgY16ewYuS5bCxV 15 | lyS28jsPht9VAgMBAAE= 16 | -----END RSA PUBLIC KEY----- 17 | dir-signing-key 18 | -----BEGIN RSA PUBLIC KEY----- 19 | MIIBigKCAYEAuMM87vcfVHbSaAK3oWvwNCxZ8fW+W5PM2hNWOyGaXF418FmOf863 20 | GW03l9wKvRPYKe0/wPaobkHZboK8rL1iSDx4CaK9EyKg7updiaKMI9Ml2XDCLYzL 21 | bRf4vlZkodgFaIBgKzhQwxq0f+yT7sbjToHD38WlE8lzCjjf+GEnSsfYJB05C083 22 | iJgKLQFHiwTt2GTR2P0oknSYN+UhFXex5EcsbcLL6dIRPONrYLEpVpKuH6YAHwJk 23 | rGyA1GdZ0QlO6tlYSJ4X3DnJPisOF/s9TB9K9GeSQqKghXHnYEDepPjgIjNvd1Ne 24 | bi0JmY0Z/qsc22E65cbYWon2g5mQCRb+MhIsQWIpQZpGqiPQ4SmHLYdiX+iHP1WS 25 | XBg9sii7aDAFUY94XQJtP8Jfg/p+kBKbw1CkCp8d/D5VdCVBvo+zZ3JD7/9fYbls 26 | P3ZjH1LhxX1ENEuwBCk8DQ3aaJiDRCLHhxsXM0B5afQhyk2AJuYiqzdKcxw527UC 27 | Rx6DW1SvY/DfAgMBAAE= 28 | -----END RSA PUBLIC KEY----- 29 | dir-key-crosscert 30 | -----BEGIN ID SIGNATURE----- 31 | W4M9vToRHli3Xhf0nzQs9T5eCs7o91c64Tkr3aiHIzyye22QsucSAErz3AvNnxfP 32 | rW3EoH76U7hdkq+v29DznRXDd1f5Ksfu1G1ZlxuQmkj2cw2eoXanjzf+JgqZ5uno 33 | AKWBeu9j05PowXCwgy5c6ok6wghCnXf3Kjng5mtBHgM9BThks8XTrbD+coi/RQwb 34 | 8lUCdy98632cV21ydo99QBENbZvSShKZ8sfphhjI5pYbRpZyT6klKuV1+MAPOV75 35 | 3amRfkXpo1q1X6Dtg+NTzknjqILTt9lXJOZvLF59aRprT2eDCwN6ra9njGpq1jOV 36 | tjKI51LLd0R8DO0ujDR/CzXr9XBZS/S/Jr1F1yIdSdGWNSEECsCZ9HajjR0Y3LJT 37 | fMNOT+kNJFyAuMcDP7ltLidoxt+1TJt5HYlmkMicwCYyKGpKtr2Q+OMFBlz/5jqR 38 | Gh2IZkqF/UTLu+dwgKzu3V/JOht6nW10ROcA+CYs1sVpQfpxNBkXbmtCbcfFVDhV 39 | -----END ID SIGNATURE----- 40 | dir-key-certification 41 | -----BEGIN SIGNATURE----- 42 | aI92/sOUwiZ8pxXOZu3iuwJ4rFhZIwfznTm+Pc9Q0uL5Zv7VyPJ56cMNYwGtTMGU 43 | WrVYk2h5lFst1/v0extmgFCIjPc6+bxohcg3E1opHL0PPjS/jHPC7i2rKufitrxO 44 | 5IjQKtUoFkPE3RfJ2YgF6X3AAZCf4XHJX6t6RQBTtfyVk4N8GA2hK3p4sVdIb9yo 45 | WQd8pEtDb6UaM3H/xz7XBfwJfjJUriPJPj0UOn3NGjWG4htwgn26I8iOXn43aovD 46 | 9gLjT8XYSmnBrcsyVPoRuLrkAUya9j7IJauScXb0lC5ff/r+R0sRKacHz/C2mRNz 47 | 0MUOZzclXGYgoEI5SlVmgjExAk1lzutHrGTU2i7gOHSyLUBkkuwBwwGRvdwin6MA 48 | dqds2KuCHuHgY+mS8cNKO2sJAFYsMZF3GgK5IhBXCeNpFWHdgsAmIvHQjKTj2BKJ 49 | LgYpBddfqew0nZauSLdutpBFXngdD9QXW2YIzCxb6pQDqBaYOygEoHQqVMWVn0BA 50 | -----END SIGNATURE----- -------------------------------------------------------------------------------- /torpy/client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import socket 17 | import logging 18 | import functools 19 | from typing import TYPE_CHECKING 20 | from contextlib import contextmanager 21 | 22 | from torpy.guard import TorGuard 23 | from torpy.utils import retry, log_retry 24 | from torpy.circuit import TorCircuit 25 | from torpy.cell_socket import TorSocketConnectError 26 | from torpy.consesus import TorConsensus 27 | from torpy.cache_storage import TorCacheDirStorage 28 | 29 | if TYPE_CHECKING: 30 | from typing import ContextManager 31 | 32 | logger = logging.getLogger(__name__) 33 | 34 | 35 | class TorClient: 36 | def __init__(self, consensus=None, auth_data=None): 37 | self._consensus = consensus or TorConsensus() 38 | self._auth_data = auth_data or {} 39 | 40 | @classmethod 41 | def create(cls, authorities=None, cache_class=None, cache_kwargs=None, auth_data=None): 42 | cache_class = cache_class or TorCacheDirStorage 43 | cache_kwargs = cache_kwargs or {} 44 | consensus = TorConsensus(authorities=authorities, cache_storage=cache_class(**cache_kwargs)) 45 | return cls(consensus, auth_data) 46 | 47 | @retry(3, BaseException, log_func=functools.partial(log_retry, 48 | msg='Retry with another guard...', 49 | no_traceback=(socket.timeout, TorSocketConnectError,)) 50 | ) 51 | def get_guard(self, by_flags=None): 52 | # TODO: add another stuff to filter guards 53 | guard_router = self._consensus.get_random_guard_node(by_flags) 54 | return TorGuard(guard_router, purpose='TorClient', consensus=self._consensus, auth_data=self._auth_data) 55 | 56 | @contextmanager 57 | def create_circuit(self, hops_count=3, guard_by_flags=None) -> 'ContextManager[TorCircuit]': 58 | with self.get_guard(guard_by_flags) as guard: 59 | yield guard.create_circuit(hops_count) 60 | 61 | def __enter__(self): 62 | """Start using the tor client.""" 63 | return self 64 | 65 | def __exit__(self, exc_type, exc_val, exc_tb): 66 | """Close the tor client.""" 67 | self.close() 68 | 69 | def close(self): 70 | self._consensus.close() 71 | -------------------------------------------------------------------------------- /torpy/parsers.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import re 17 | import logging 18 | 19 | from torpy.crypto_common import b64decode 20 | 21 | logger = logging.getLogger(__name__) 22 | 23 | 24 | class HSDescriptorParser: 25 | regex = re.compile( 26 | """\ 27 | introduction-points 28 | -----BEGIN MESSAGE----- 29 | (.+?) 30 | -----END MESSAGE-----""", 31 | flags=re.DOTALL | re.IGNORECASE, 32 | ) 33 | 34 | @staticmethod 35 | def parse(data): 36 | m = __class__.regex.search(data) 37 | if m: 38 | return m.group(1) 39 | else: 40 | logger.error("Can't parse HSDescriptor: %r", data) 41 | raise Exception("Can't parse HSDescriptor") 42 | 43 | 44 | class RouterDescriptorParser: 45 | regex = re.compile( 46 | r"""\ 47 | onion-key 48 | -----BEGIN RSA PUBLIC KEY----- 49 | (?P.+?) 50 | -----END RSA PUBLIC KEY----- 51 | signing-key 52 | -----BEGIN RSA PUBLIC KEY----- 53 | (?P.+?) 54 | -----END RSA PUBLIC KEY----- 55 | .+? 56 | ntor-onion-key (?P[^\n]+)""", 57 | flags=re.DOTALL | re.IGNORECASE, 58 | ) 59 | 60 | @staticmethod 61 | def parse(data): 62 | m = __class__.regex.search(data) 63 | if m: 64 | return {k: b64decode(v) for k, v in m.groupdict().items()} 65 | else: 66 | logger.debug("Can't parse router descriptor: %r", data) 67 | raise Exception("Can't parse router descriptor") 68 | 69 | 70 | class IntroPointParser: 71 | regex = re.compile( 72 | r"""\ 73 | introduction-point (?P[^\n]+) 74 | ip-address (?P[^\n]+) 75 | onion-port (?P[0-9]+) 76 | onion-key 77 | -----BEGIN RSA PUBLIC KEY----- 78 | (?P.+?) 79 | -----END RSA PUBLIC KEY----- 80 | service-key 81 | -----BEGIN RSA PUBLIC KEY----- 82 | (?P.+?) 83 | -----END RSA PUBLIC KEY-----""", 84 | flags=re.DOTALL | re.IGNORECASE, 85 | ) 86 | 87 | @staticmethod 88 | def _decode(d): 89 | for k in ('onion_key', 'service_key'): 90 | d[k] = b64decode(d[k]) 91 | return d 92 | 93 | @staticmethod 94 | def parse(data): 95 | res = [__class__._decode(m.groupdict()) for m in __class__.regex.finditer(data)] 96 | return res 97 | -------------------------------------------------------------------------------- /torpy/cli/console.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import logging 17 | import textwrap 18 | from argparse import ArgumentParser 19 | 20 | from torpy.utils import register_logger 21 | from torpy.http.urlopener import do_request as urllib_request 22 | try: 23 | from torpy.http.requests import do_request as requests_request 24 | except ImportError: 25 | requests_request = None 26 | 27 | logger = logging.getLogger(__name__) 28 | 29 | 30 | def print_data(data, to_file=None): 31 | if to_file: 32 | logger.info('Writing to file %s', to_file) 33 | with open(to_file, 'w+') as f: 34 | f.write(data) 35 | else: 36 | logger.warning(textwrap.indent(data, '> ', lambda line: True)) 37 | 38 | 39 | def main(): 40 | parser = ArgumentParser() 41 | parser.add_argument('--url', help='url', required=True) 42 | parser.add_argument('--method', default='GET', type=str.upper, help='http method') 43 | parser.add_argument('--data', default=None, help='http data') 44 | parser.add_argument('--hops', default=3, help='hops count', type=int) 45 | parser.add_argument('--to-file', default=None, help='save result to file') 46 | parser.add_argument('--header', default=None, dest='headers', nargs=2, action='append', help='set some http header') 47 | parser.add_argument('--auth-data', nargs=2, action='append', help='set auth data for hidden service authorization') 48 | parser.add_argument('--log-file', default=None, help='log file path') 49 | parser.add_argument('--requests-lib', dest='request_func', default=urllib_request, action='store_const', 50 | const=requests_request, help='use requests library for making requests') 51 | parser.add_argument('-v', '--verbose', default=0, help='enable verbose output', action='count') 52 | args = parser.parse_args() 53 | 54 | register_logger(args.verbose, log_file=args.log_file) 55 | 56 | if not args.request_func: 57 | raise Exception('Requests library not installed, use default urllib') 58 | 59 | data = args.request_func(args.url, method=args.method, data=args.data, headers=args.headers, hops=args.hops, 60 | auth_data=args.auth_data, verbose=args.verbose) 61 | print_data(data, args.to_file) 62 | 63 | 64 | if __name__ == '__main__': 65 | try: 66 | main() 67 | except KeyboardInterrupt: 68 | logger.error('Interrupted.') 69 | -------------------------------------------------------------------------------- /torpy/http/requests.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import logging 17 | from contextlib import contextmanager 18 | 19 | from requests import Request, Session 20 | 21 | try: 22 | from urllib3.util import SKIP_HEADER 23 | except Exception: 24 | SKIP_HEADER = None 25 | 26 | from torpy.client import TorClient 27 | from torpy.http.adapter import TorHttpAdapter 28 | 29 | logger = logging.getLogger(__name__) 30 | 31 | 32 | class TorRequests: 33 | def __init__(self, hops_count=3, headers=None, auth_data=None): 34 | self._hops_count = hops_count 35 | self._headers = dict(headers) if headers else {} 36 | self._auth_data = dict(auth_data) if auth_data else auth_data 37 | 38 | def __enter__(self): 39 | """Create TorClient and connect to guard node.""" 40 | self._tor = TorClient(auth_data=self._auth_data) 41 | self._guard = self._tor.get_guard() 42 | return self 43 | 44 | def __exit__(self, exc_type, exc_val, exc_tb): 45 | """Close guard connection.""" 46 | self._guard.close() 47 | self._tor.close() 48 | 49 | def send(self, method, url, data=None, **kwargs): 50 | with self.get_session() as s: 51 | r = Request(method, url, data, **kwargs) 52 | return s.send(r.prepare()) 53 | 54 | @contextmanager 55 | def get_session(self, retries=0): 56 | adapter = TorHttpAdapter(self._guard, self._hops_count, retries=retries) 57 | with Session() as s: 58 | s.headers.update(self._headers) 59 | s.mount('http://', adapter) 60 | s.mount('https://', adapter) 61 | yield s 62 | 63 | 64 | @contextmanager 65 | def tor_requests_session(hops_count=3, headers=None, auth_data=None, retries=0): 66 | with TorRequests(hops_count, headers, auth_data) as tr: 67 | with tr.get_session(retries=retries) as s: 68 | yield s 69 | 70 | 71 | def do_request(url, method='GET', data=None, headers=None, hops=3, auth_data=None, verbose=0, retries=0): 72 | with tor_requests_session(hops, auth_data, retries=retries) as s: 73 | headers = dict(headers or []) 74 | # WARN: https://github.com/urllib3/urllib3/pull/1750 75 | if SKIP_HEADER and \ 76 | 'user-agent' not in (k.lower() for k in headers.keys()): 77 | headers['User-Agent'] = SKIP_HEADER 78 | request = Request(method, url, data=data, headers=headers) 79 | logger.warning('Sending: %s %s', request.method, request.url) 80 | response = s.send(request.prepare()) 81 | logger.warning('Response status: %r', response.status_code) 82 | return response.text 83 | -------------------------------------------------------------------------------- /tests/unittest/data/network_status/consensus: -------------------------------------------------------------------------------- 1 | network-status-version 3 2 | vote-status consensus 3 | consensus-method 26 4 | valid-after 2017-05-25 04:46:30 5 | fresh-until 2017-05-25 04:46:40 6 | valid-until 2017-05-25 04:46:50 7 | voting-delay 2 2 8 | client-versions 9 | server-versions 10 | known-flags Authority Exit Fast Guard HSDir NoEdConsensus Running Stable V2Dir Valid 11 | recommended-client-protocols Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2 12 | recommended-relay-protocols Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2 13 | required-client-protocols Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2 14 | required-relay-protocols Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=3-4 LinkAuth=1 Microdesc=1 Relay=1-2 15 | dir-source test001a 596CD48D61FDA4E868F4AA10FF559917BE3B1A35 127.0.0.1 127.0.0.1 7001 5001 16 | contact auth1@test.test 17 | vote-digest 2E7177224BBA39B505F7608FF376C07884CF926F 18 | dir-source test000a BCB380A633592C218757BEE11E630511A485658A 127.0.0.1 127.0.0.1 7000 5000 19 | contact auth0@test.test 20 | vote-digest 5DD41617166FFB82882A117EEFDA0353A2794DC5 21 | r test002r NIIl+DyFR5ay3WNk5lyxibM71pY UzQp+EE8G0YCKtNlZVy+3h5tv0Q 2017-05-25 04:46:11 127.0.0.1 5002 7002 22 | s Exit Fast Guard HSDir Running Stable V2Dir Valid 23 | v Tor 0.3.0.7 24 | pr Cons=1-2 Desc=1-2 DirCache=1 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 25 | w Bandwidth=0 Unmeasured=1 26 | p accept 1-65535 27 | r test001a qgzRpIKSW809FnL4tntRtWgOiwo x8yR5mi/DBbLg46qwGQ96Dno+nc 2017-05-25 04:46:12 127.0.0.1 5001 7001 28 | s Authority Exit Fast Guard HSDir Running V2Dir Valid 29 | v Tor 0.3.0.7 30 | pr Cons=1-2 Desc=1-2 DirCache=1 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 31 | w Bandwidth=0 Unmeasured=1 32 | p reject 1-65535 33 | r test000a 3nJC+LvtNmx6kw23x1WE90pyIj4 Hg3NyPqDZoRQN8hVI5Vi6B+pofw 2017-05-25 04:46:12 127.0.0.1 5000 7000 34 | s Authority Exit Fast Guard HSDir Running Stable V2Dir Valid 35 | v Tor 0.3.0.7 36 | pr Cons=1-2 Desc=1-2 DirCache=1 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 37 | w Bandwidth=0 Unmeasured=1 38 | p reject 1-65535 39 | directory-footer 40 | bandwidth-weights Wbd=3333 Wbe=0 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=3333 Wee=10000 Weg=3333 Wem=10000 Wgb=10000 Wgd=3333 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=3333 Wme=0 Wmg=0 Wmm=10000 41 | directory-signature 596CD48D61FDA4E868F4AA10FF559917BE3B1A35 9FBF54D6A62364320308A615BF4CF6B27B254FAD 42 | -----BEGIN SIGNATURE----- 43 | Ho0rLojfLHs9cSPFxe6znuGuFU8BvRr6gnH1gULTjUZO0NSQvo5N628KFeAsq+pT 44 | ElieQeV6UfwnYN1U2tomhBYv3+/p1xBxYS5oTDAITxLUYvH4pLYz09VutwFlFFtU 45 | r/satajuOMST0M3wCCBC4Ru5o5FSklwJTPJ/tWRXDCEHv/N5ZUUkpnNdn+7tFSZ9 46 | eFrPxPcQvB05BESo7C4/+ZnZVO/wduObSYu04eWwTEog2gkSWmsztKoXpx1QGrtG 47 | sNL22Ws9ySGDO/ykFFyxkcuyB5A8oPyedR7DrJUfCUYyB8o+XLNwODkCFxlmtFOj 48 | ci356fosgLiM1sVqCUkNdA== 49 | -----END SIGNATURE----- 50 | directory-signature BCB380A633592C218757BEE11E630511A485658A 9CA027E05B0CE1500D90DA13FFDA8EDDCD40A734 51 | -----BEGIN SIGNATURE----- 52 | uiAt8Ir27pYFX5fNKiVZDoa6ELVEtg/E3YeYHAnlSSRzpacLMMTN/HhF//Zvv8Zj 53 | FKT95v77xKvE6b8s7JjB3ep6coiW4tkLqjDiONG6iDRKBmy6D+RZgf1NMxl3gWaZ 54 | ShINORJMW9nglnBbysP7egPiX49w1igVZQLM1C2ppphK6uO5EGcK6nDJF4LVDJ7B 55 | Fvt2yhY+gsiG3oSrhsP0snQnFfvEeUFO/r2gRVJ1FoMXUttaOCtmj268xS08eZ0m 56 | MS+u6gHEM1dYkwpA+LzE9G4akPRhvRjMDaF0RMuLQ7pY5v44uE5OX5n/GKWRgzVZ 57 | DH+ubl6BuqpQxYQXaHZ5iw== 58 | -----END SIGNATURE----- 59 | -------------------------------------------------------------------------------- /tests/unittest/data/network_status/consensus_extra: -------------------------------------------------------------------------------- 1 | network-status-version 3 5 2 | vote-status consensus 3 | consensus-method 26 4 | valid-after 2017-05-25 04:46:30 strange 5 | fresh-until 2017-05-25 04:46:40 6 | valid-until 2017-05-25 04:46:50 7 | voting-delay 2 2 8 | client-versions 9 | server-versions 10 | known-flags Authority Exit Fast Guard NewFlag1 HSDir NoEdConsensus Running Stable V2Dir Valid NewFlag2 11 | recommended-client-protocols Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2 12 | recommended-relay-protocols Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2 13 | required-client-protocols Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2 14 | required-relay-protocols Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=3-4 LinkAuth=1 Microdesc=1 Relay=1-2 15 | dir-source test001a 596CD48D61FDA4E868F4AA10FF559917BE3B1A35 127.0.0.1 127.0.0.1 7001 5001 extra_arg1 extra_arg2 16 | contact auth1@test.test 17 | extra-keyword 111 222 18 | vote-digest 2E7177224BBA39B505F7608FF376C07884CF926F AAAAAAA 19 | dir-source test000a BCB380A633592C218757BEE11E630511A485658A 127.0.0.1 127.0.0.1 7000 5000 20 | contact auth0@test.test 21 | vote-digest 5DD41617166FFB82882A117EEFDA0353A2794DC5 22 | extra-keyword 333 444 23 | r test002r NIIl+DyFR5ay3WNk5lyxibM71pY UzQp+EE8G0YCKtNlZVy+3h5tv0Q 2017-05-25 04:46:11 127.0.0.1 5002 7002 24 | s Exit Fast Guard HSDir Running Stable V2Dir Valid NewFlag 25 | v Tor 0.3.0.7 26 | pr Cons=1-2 Desc=1-2 DirCache=1 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 27 | w Bandwidth=0 Unmeasured=1 Strange=2 28 | p accept 1-65535 29 | r test001a qgzRpIKSW809FnL4tntRtWgOiwo x8yR5mi/DBbLg46qwGQ96Dno+nc 2017-05-25 04:46:12 127.0.0.1 5001 7001 30 | s Authority Exit Fast Guard HSDir Running V2Dir Valid 31 | v Tor 0.3.0.7 32 | pr Cons=1-2 Desc=1-2 DirCache=1 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 33 | w Bandwidth=0 Unmeasured=1 34 | p reject 1-65535 35 | r test000a 3nJC+LvtNmx6kw23x1WE90pyIj4 Hg3NyPqDZoRQN8hVI5Vi6B+pofw 2017-05-25 04:46:12 127.0.0.1 5000 7000 36 | s Authority Exit Fast Guard HSDir Running Stable V2Dir Valid 37 | v Tor 0.3.0.7 38 | pr Cons=1-2 Desc=1-2 DirCache=1 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 39 | w Bandwidth=0 Unmeasured=1 40 | p reject 1-65535 41 | directory-footer 42 | bandwidth-weights Wbd=3333 Wbe=0 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=3333 Wee=10000 Weg=3333 Wem=10000 Wgb=10000 Wgd=3333 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=3333 Wme=0 Wmg=0 Wmm=10000 43 | directory-signature 596CD48D61FDA4E868F4AA10FF559917BE3B1A35 9FBF54D6A62364320308A615BF4CF6B27B254FAD 44 | -----BEGIN SIGNATURE----- 45 | Ho0rLojfLHs9cSPFxe6znuGuFU8BvRr6gnH1gULTjUZO0NSQvo5N628KFeAsq+pT 46 | ElieQeV6UfwnYN1U2tomhBYv3+/p1xBxYS5oTDAITxLUYvH4pLYz09VutwFlFFtU 47 | r/satajuOMST0M3wCCBC4Ru5o5FSklwJTPJ/tWRXDCEHv/N5ZUUkpnNdn+7tFSZ9 48 | eFrPxPcQvB05BESo7C4/+ZnZVO/wduObSYu04eWwTEog2gkSWmsztKoXpx1QGrtG 49 | sNL22Ws9ySGDO/ykFFyxkcuyB5A8oPyedR7DrJUfCUYyB8o+XLNwODkCFxlmtFOj 50 | ci356fosgLiM1sVqCUkNdA== 51 | -----END SIGNATURE----- 52 | directory-signature BCB380A633592C218757BEE11E630511A485658A 9CA027E05B0CE1500D90DA13FFDA8EDDCD40A734 53 | -----BEGIN SIGNATURE----- 54 | uiAt8Ir27pYFX5fNKiVZDoa6ELVEtg/E3YeYHAnlSSRzpacLMMTN/HhF//Zvv8Zj 55 | FKT95v77xKvE6b8s7JjB3ep6coiW4tkLqjDiONG6iDRKBmy6D+RZgf1NMxl3gWaZ 56 | ShINORJMW9nglnBbysP7egPiX49w1igVZQLM1C2ppphK6uO5EGcK6nDJF4LVDJ7B 57 | Fvt2yhY+gsiG3oSrhsP0snQnFfvEeUFO/r2gRVJ1FoMXUttaOCtmj268xS08eZ0m 58 | MS+u6gHEM1dYkwpA+LzE9G4akPRhvRjMDaF0RMuLQ7pY5v44uE5OX5n/GKWRgzVZ 59 | DH+ubl6BuqpQxYQXaHZ5iw== 60 | -----END SIGNATURE----- 61 | -------------------------------------------------------------------------------- /torpy/documents/network_status_diff.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 James Brown 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | from enum import Enum 17 | 18 | from torpy.documents import TorDocument 19 | from torpy.documents.items import ItemMask, ItemInt, Item, ItemParsers 20 | 21 | 22 | class EdActionType(Enum): 23 | Delete = 'd' 24 | Append = 'a' 25 | Change = 'c' 26 | 27 | 28 | class EdAction: 29 | def __init__(self, start, end, act_type, data): 30 | self.start = start 31 | self.end = end 32 | self.type = act_type 33 | self.data = data 34 | 35 | def apply(self, lines, cur_line, cur_end): 36 | start, end = self.start, self.end 37 | if start == '$': 38 | start = end = cur_line 39 | if end == '$': 40 | end = cur_end 41 | 42 | if self.type == EdActionType.Append: 43 | lines[start:start] = self.data 44 | return start + len(self.data) 45 | 46 | if self.type == EdActionType.Change: 47 | lines[start - 1:end] = self.data 48 | return start - 1 + len(self.data) 49 | 50 | if self.type == EdActionType.Delete: 51 | del lines[start - 1:end] 52 | return start 53 | 54 | 55 | class ItemAction(ItemMask): 56 | @staticmethod 57 | def _parse_action(line, m, lines, *_): 58 | if m is None: 59 | raise ValueError(f'ed line contains invalid command: {line.rstrip()}') 60 | parts = m.groupdict() 61 | start = int(parts['start']) 62 | if start > 2147483647: 63 | raise ValueError(f'ed line contains line number > INT32_MAX: {start}') 64 | end = parts['end'] 65 | if end is None: 66 | end = start 67 | elif end == '$': 68 | end = '$' 69 | else: 70 | end = int(end) 71 | if end > 2147483647: 72 | raise ValueError(f'ed line contains line number > INT32_MAX: {end}') 73 | if end < start: 74 | raise ValueError(f'ed line contains invalid range: ({start}, {end})') 75 | action = EdActionType(parts['action']) 76 | 77 | data = [] 78 | if action in (EdActionType.Append, EdActionType.Change): 79 | for line in lines: 80 | if line == '.': 81 | break 82 | data.append(line) 83 | 84 | return EdAction(start, end, action, data) 85 | 86 | def __init__(self, out_name): 87 | super().__init__( 88 | r'(?P\d+)(?:,(?P\d+|\$))?(?P[acd])', self._parse_action, out_name, as_list=True 89 | ) 90 | 91 | 92 | class NetworkStatusDiffDocument(TorDocument): 93 | DOCUMENT_NAME = 'network_status_diff' 94 | 95 | # The first line is "network-status-diff-version 1" NL 96 | START_ITEM = ItemInt('network-status-diff-version') 97 | 98 | ITEMS = [ 99 | # The second line is "hash" SP FromDigest SP ToDigest NL 100 | Item('hash', parse_func=ItemParsers.split_symbol, parse_args=[' ', ['from_digest', 'to_digest']]), 101 | # Diff Actions 102 | ItemAction(out_name='actions'), 103 | ] 104 | 105 | def __init__(self, raw_string): 106 | super().__init__(raw_string) 107 | -------------------------------------------------------------------------------- /tests/unittest/test_consensus.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from torpy.consesus import DirectoryServer, DirectoryFlags, RouterFlags 4 | 5 | 6 | @pytest.mark.parametrize( 7 | 'line, result', 8 | [ 9 | ( 10 | '"moria1 orport=9101 v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31"', # noqa: E501, E126 11 | {'_nickname': 'moria1', '_fingerprint': b'\x96\x95\xdf\xc3_\xfe\xb8a2\x9b\x9f\x1a\xb0LF9p \xce1', 12 | '_digest': None, '_ip': '128.31.0.39', '_or_port': 9101, '_dir_port': 9131, '_version': None, 13 | '_flags': [RouterFlags.Authority], '_consensus': None, '_service_key': None, 14 | '_dir_flags': DirectoryServer.AUTH_FLAGS, '_v3ident': 'D586D18309DED4CD6D57C18FDB97EFA96D330566', 15 | '_ipv6': None, 16 | '_bridge': False}), 17 | ( 18 | '"tor26 orport=443 v3ident=14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 ipv6=[2001:858:2:2:aabb:0:563b:1526]:443 86.59.21.38:80 847B 1F85 0344 D787 6491 A548 92F9 0493 4E4E B85D"', # noqa: E501 19 | {'_nickname': 'tor26', '_fingerprint': b'\x84{\x1f\x85\x03D\xd7\x87d\x91\xa5H\x92\xf9\x04\x93NN\xb8]', 20 | '_digest': None, '_ip': '86.59.21.38', '_or_port': 443, '_dir_port': 80, '_version': None, 21 | '_flags': [RouterFlags.Authority], '_consensus': None, '_service_key': None, 22 | '_dir_flags': DirectoryServer.AUTH_FLAGS, '_v3ident': '14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4', 23 | '_ipv6': '[2001:858:2:2:aabb:0:563b:1526]:443', '_bridge': False}), 24 | ( 25 | '"dizum orport=443 v3ident=E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58 45.66.33.45:80 7EA6 EAD6 FD83 083C 538F 4403 8BBF A077 587D D755"', # noqa: E501 26 | {'_nickname': 'dizum', '_fingerprint': b'~\xa6\xea\xd6\xfd\x83\x08