├── thor ├── py.typed ├── http │ ├── client │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── initiate.py │ │ ├── client.py │ │ ├── connection.py │ │ └── exchange.py │ ├── __init__.py │ ├── uri.py │ ├── error.py │ └── server.py ├── __init__.py ├── dns │ └── __init__.py ├── tls.py ├── udp.py ├── events.py ├── tcp.py └── loop.py ├── MANIFEST.in ├── .gitignore ├── .coveragerc ├── doc ├── README.md ├── tls.md ├── udp.md ├── loop.md ├── events.md ├── tcp.md └── http.md ├── .editorconfig ├── .github ├── dependabot.yml └── workflows │ ├── ci.yml │ ├── codeql-analysis.yml │ └── publish.yml ├── Makefile ├── LICENSE.md ├── test ├── test_http_utils.py ├── test.cert ├── test.key ├── test_tcp_server.py ├── test_dns.py ├── test_udp.py ├── test_graceful.py ├── test_events.py ├── test_tls_client.py ├── framework.py ├── test_loop.py ├── test_tcp_client.py ├── test_http_server.py └── test_http_parser.py ├── pyproject.toml ├── README.md ├── Makefile.pyproject └── Makefile.venv /thor/py.typed: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include thor/py.typed 2 | recursive-include doc *.md 3 | -------------------------------------------------------------------------------- /thor/http/client/__init__.py: -------------------------------------------------------------------------------- 1 | from .client import HttpClient 2 | from .exchange import HttpClientExchange 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | __pycache__ 3 | .DS_Store 4 | /MANIFEST 5 | /dist 6 | /build 7 | /changelog.md 8 | /.coverage 9 | /.mypy_cache 10 | /.pytest_cache 11 | /.venv 12 | /*.egg-info 13 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = thor 3 | 4 | [report] 5 | exclude_lines = 6 | pragma: no cover 7 | def __repr__ 8 | raise AssertionError 9 | raise NotImplementedError 10 | if __name__ == .__main__.: 11 | raise -------------------------------------------------------------------------------- /thor/http/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from __future__ import absolute_import 4 | from thor.http.client import HttpClient 5 | from thor.http.server import HttpServer 6 | from thor.http.common import ( 7 | header_names, 8 | header_dict, 9 | get_header, 10 | safe_methods, 11 | idempotent_methods, 12 | hop_by_hop_hdrs, 13 | ) 14 | -------------------------------------------------------------------------------- /doc/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Thor - Easy Evented Intermediation 3 | 4 | ## API Reference 5 | 6 | * [Events](events.md) - Emitting and listening for events 7 | * [The Loop](loop.md) - The event loop itself 8 | * [TCP](tcp.md) - Network connections 9 | * [TLS/SSL](tls.md) - Encrypted network connections 10 | * [UDP](udp.md) - Network datagrams 11 | * [HTTP](http.md) - HyperText Transfer Protocol -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # top-most EditorConfig file 2 | root = true 3 | 4 | # Unix-style newlines with a newline ending every file 5 | [*] 6 | end_of_line = lf 7 | insert_final_newline = true 8 | charset = utf-8 9 | max_line_length = 100 10 | trim_trailing_whitespace = true 11 | indent_style = space 12 | indent_size = 4 13 | 14 | [Makefile*] 15 | indent_style = tab 16 | 17 | [{*.md,*.js,*.scss,*.yml}] 18 | indent_size = 2 19 | 20 | [*.html] 21 | max_line_length = unset 22 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "github-actions" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "weekly" 12 | -------------------------------------------------------------------------------- /doc/tls.md: -------------------------------------------------------------------------------- 1 | # TLS/SSL 2 | 3 | ## thor.TlsClient ( _[thor.loop](loop.md)_ `loop`? ) 4 | 5 | A TCP client with a SSL/TLS wrapper. The interface is identical to that 6 | of [thor.TcpClient](tcp.md#thortcpclient--thorloop-loop-). 7 | 8 | For example: 9 | 10 | import sys 11 | import thor 12 | 13 | test_host, test_port = sys.argv[1:2] 14 | 15 | def handle_connect(conn): 16 | conn.on('data', sys.stdout.write) 17 | conn.on('close', thor.stop) 18 | conn.write("GET /\n\n") 19 | conn.pause(False) 20 | 21 | def handle_err(err_type, err): 22 | sys.stderr.write(str(err_type)) 23 | thor.stop() 24 | 25 | c = thor.TlsClient() 26 | c.on('connect', handle_connect) 27 | c.on('connect_error', handle_err) 28 | c.connect(test_host, test_port) 29 | thor.run() 30 | 31 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | 7 | pull_request: 8 | branches: [ main ] 9 | 10 | jobs: 11 | test: 12 | strategy: 13 | matrix: 14 | python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] 15 | os: ["ubuntu-latest"] 16 | runs-on: ${{ matrix.os }} 17 | 18 | steps: 19 | - uses: actions/checkout@v6 20 | with: 21 | submodules: 'true' 22 | - name: Set up Python ${{ matrix.python-version }} 23 | uses: actions/setup-python@v6 24 | with: 25 | python-version: ${{ matrix.python-version }} 26 | - name: Cache venv 27 | uses: actions/cache@v5 28 | with: 29 | path: .venv 30 | key: ${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('pyproject.toml') }} 31 | - name: Set up venv 32 | run: make venv 33 | - name: Typecheck 34 | run: make typecheck 35 | - name: Test 36 | run: make -e test 37 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJECT=thor 2 | GITHUB_STEP_SUMMARY ?= throwaway 3 | 4 | 5 | ########################################################################################## 6 | ## Tests 7 | 8 | .PHONY: test 9 | test: venv 10 | PYTHONPATH=.:$(VENV) $(VENV)/pytest --md $(GITHUB_STEP_SUMMARY) -v -n auto test 11 | rm -f throwaway 12 | 13 | .PHONY: test/*.py 14 | test/*.py: venv 15 | PYTHONPATH=.:$(VENV) $(VENV)/pytest -v $@ 16 | 17 | 18 | ############################################################################# 19 | ## Tasks 20 | 21 | .PHONY: cli 22 | cli: venv 23 | PYTHONPATH=$(VENV) $(VENV)/pip install . 24 | PYTHONPATH=$(VENV):. sh 25 | 26 | .PHONY: clean 27 | clean: clean_py 28 | 29 | .PHONY: tidy 30 | tidy: tidy_py 31 | 32 | .PHONY: lint 33 | lint: lint_py 34 | 35 | .PHONY: typecheck 36 | typecheck: typecheck_py 37 | 38 | .PHONY: loop_type 39 | loop_type: 40 | PYTHONPATH=$(VENV) $(VENV)/python -c "import thor.loop; print(thor.loop._loop.__class__)" 41 | 42 | 43 | include Makefile.pyproject 44 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # License 2 | 3 | 4 | Copyright (c) 2005– Mark Nottingham 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in 14 | all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /doc/udp.md: -------------------------------------------------------------------------------- 1 | # UDP 2 | 3 | 4 | ## thor.UdpEndpoint ( _[thor.loop](loop.md)_ `loop`? ) 5 | 6 | A UDP endpoint. If `loop` is omitted, the "default" loop will be used. 7 | 8 | Note that new endpoints will not emit *datagram* events until they are unpaused; see [thor.UdpEndpoint.pause](#void-thorudpendpointpause-bool-paused-). 9 | 10 | 11 | ### _int_ thor.UdpEndpoint.max\_dgram 12 | 13 | The maximum number of bytes that sent with *send()*. 14 | 15 | 16 | ### _void_ thor.UdpEndpoint.bind ( _str_ `host`, _int_ `port` ) 17 | 18 | Optionally binds the endpoint to _port_ on _host_ (which must be a local interface). If called, it must occur before *send()*. 19 | 20 | If not called before *send()*, the socket will be assigned a random local port by the operating system. 21 | 22 | 23 | ### _void_ thor.UdpEndpoint.send ( _bytes_ `datagram`, _str_ `host`, _int_ `port` ) 24 | 25 | Send `datagram` to `port` on `host`. 26 | 27 | Note that UDP is intrinsically an unreliable protocol, so the datagram may or may not be received. See also *thor.UdpEndpoint.max\_dgram.* 28 | 29 | 30 | ### _void_ thor.UdpEndpoint.pause ( _bool_ `paused` ) 31 | 32 | Stop the endpoint from emitting *datagram* events if `paused` is True; resume emitting them if False. 33 | 34 | 35 | ### _void_ thor.UdpEndpoint.shutdown () 36 | 37 | Stop the endpoint. 38 | 39 | 40 | ### event 'datagram' ( _bytes_ `datagram`, _str_ `host`, _int_ `port` ) 41 | 42 | Emitted when the socket receives `datagram` from `port` on `host`. -------------------------------------------------------------------------------- /test/test_http_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import unittest 5 | 6 | from thor.http.common import header_names, header_dict, get_header 7 | 8 | 9 | class TestHttpUtilsBytes(unittest.TestCase): 10 | hdrs = [ 11 | (b"A", b"a1"), 12 | (b"B", b"b1"), 13 | (b"a", b"a2"), 14 | (b"C", b"c1"), 15 | (b"b", b"b2"), 16 | (b"A", b"a3, a4"), 17 | (b"D", b'"d1, d1"'), 18 | ] 19 | 20 | def test_header_names(self): 21 | hdrs_n = header_names(self.hdrs) 22 | self.assertEqual(hdrs_n, set([b"a", b"b", b"c", b"d"])) 23 | 24 | def test_header_dict(self): 25 | hdrs_d = header_dict(self.hdrs) 26 | self.assertEqual(hdrs_d[b"a"], [b"a1", b"a2", b"a3", b"a4"]) 27 | self.assertEqual(hdrs_d[b"b"], [b"b1", b"b2"]) 28 | self.assertEqual(hdrs_d[b"c"], [b"c1"]) 29 | 30 | def test_header_dict_omit(self): 31 | hdrs_d = header_dict(self.hdrs, b"b") 32 | self.assertEqual(hdrs_d[b"a"], [b"a1", b"a2", b"a3", b"a4"]) 33 | self.assertTrue(b"b" not in list(hdrs_d)) 34 | self.assertTrue(b"B" not in list(hdrs_d)) 35 | self.assertEqual(hdrs_d[b"c"], [b"c1"]) 36 | 37 | def test_get_header(self): 38 | self.assertEqual(get_header(self.hdrs, b"a"), [b"a1", b"a2", b"a3", b"a4"]) 39 | self.assertEqual(get_header(self.hdrs, b"b"), [b"b1", b"b2"]) 40 | self.assertEqual(get_header(self.hdrs, b"c"), [b"c1"]) 41 | 42 | 43 | if __name__ == "__main__": 44 | unittest.main() 45 | -------------------------------------------------------------------------------- /thor/http/client/__main__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from typing import Callable 3 | from thor.events import on 4 | from thor.http.common import RawHeaderListType 5 | from thor.http.error import HttpError 6 | from thor.loop import stop, run, schedule 7 | from .client import HttpClient 8 | 9 | 10 | def test_client( 11 | request_uri: bytes, out: Callable, err: Callable 12 | ) -> None: # pragma: no coverage 13 | "A simple demonstration of a client." 14 | cl = HttpClient() 15 | cl.connect_timeout = 5 16 | cl.careful = False 17 | ex = cl.exchange() 18 | 19 | @on(ex) 20 | def response_start( 21 | status: bytes, phrase: bytes, headers: RawHeaderListType 22 | ) -> None: 23 | "Print the response headers." 24 | out(b"HTTP/%s %s %s\n" % (ex.res_version, status, phrase)) 25 | out(b"\n".join([b"%s:%s" % header for header in headers])) 26 | print() 27 | print() 28 | 29 | @on(ex) 30 | def error(err_msg: HttpError) -> None: 31 | if err_msg: 32 | err(f"\033[1;31m*** ERROR:\033[0;39m {err_msg.desc} ({err_msg.detail})\n") 33 | if not err_msg.client_recoverable: 34 | stop() 35 | 36 | ex.on("response_body", out) 37 | 38 | @on(ex) 39 | def response_done(trailers: RawHeaderListType) -> None: 40 | schedule(1, stop) 41 | 42 | ex.request_start(b"GET", request_uri, []) 43 | ex.request_done([]) 44 | run() 45 | 46 | 47 | if __name__ == "__main__": 48 | test_client(sys.argv[1].encode("ascii"), sys.stdout.buffer.write, sys.stderr.write) 49 | -------------------------------------------------------------------------------- /test/test.cert: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIEQzCCAqugAwIBAgIRAPP4v1FjmVijTvfS4+v+U+swDQYJKoZIhvcNAQELBQAw 3 | aTEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMR8wHQYDVQQLDBZtbm90 4 | QEF0dGl0dWRlLUFkanVzdGVyMSYwJAYDVQQDDB1ta2NlcnQgbW5vdEBBdHRpdHVk 5 | ZS1BZGp1c3RlcjAeFw0xOTA2MDEwMDAwMDBaFw0yOTA4MTkwNTQzMzlaMFwxJzAl 6 | BgNVBAoTHm1rY2VydCBkZXZlbG9wbWVudCBjZXJ0aWZpY2F0ZTExMC8GA1UECwwo 7 | bW5vdEBBdHRpdHVkZS1BZGp1c3RlciAoTWFyayBOb3R0aW5naGFtKTCCASIwDQYJ 8 | KoZIhvcNAQEBBQADggEPADCCAQoCggEBAKcDr4wF9wYWV3LPeW1r2cchWScQ8HBI 9 | ctmOK/5Sr53lZWg0HvYPEs9My0sZhPopF5x4pXOW8+f3KPEZFdEagZEjEi1PfUfx 10 | IRaf2x/aH8M/NkYzCOtx+IerA2zX8z4Pw/P0fxplKkjfWeWCgC4zNCPcH80YxV/K 11 | N57LKJLdf1SrYkn11NiE5Xqoe/vU3b26WELXZcfd0tyF96fK1nw4sJ9z8rkOunTp 12 | +00mnUw/bZdJkzY9UUGskeLFwHbzuV379tF+OF7PnBA7bNu1O/niWwemVhrWo/ug 13 | oq9cIfDwBOmWvuacTqeAo+Vmt6sqPOwauhw5/Kpg1OnIi1PBS+XOwxECAwEAAaNz 14 | MHEwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB 15 | /wQCMAAwHwYDVR0jBBgwFoAUyGQOGlrFrmCJ0rUgSw3Q0tn4tswwGwYDVR0RBBQw 16 | EoIQdGVzdC5leGFtcGxlLm9yZzANBgkqhkiG9w0BAQsFAAOCAYEAvgnezDIkwm0g 17 | 1nwmXotvE7ftD2FY9oPhpeRpm1fH02KI6/noGmNcrcc0hSVRdrp8MEYXRv6evs2R 18 | mOEL0H+zBfyhmupYKHo72zr721IUIA5gTuYsK791wcSi/mIoI5gmeWA8qqOmsQ4r 19 | co0LJoJMB6bluBki/xOhtRKduk0ALBE/oSfn9T0OIpEUm4RzyxzFZ+GsZCXCSUkz 20 | GYnjEPSM7sRNFUpvCXA1hYG5kgy1htfSZqDmQwIM/gQy0luoHSfVkkHkByTTtN2y 21 | m+x9XkASFOyiuq35P0Tg+zEpa76AmzBNA/LOJFJ/NAu0c5RnsEUnJ0DP6jDvEFa5 22 | 2Ss6NuOEGj4ffiNdTxmNHBl+X6W3HBw4OiES2BgeNlcEI6e90dPPTeeftGx0z7/2 23 | 4HOYp15iImbtHVfck47C7uEWY2WbamG06VRommFBcA3m4tveEm153rohsgn6qwQL 24 | qqPREnRtFcvMKDeE5M2Jl9zNwsRIkn2pj136HNU0wWXtpIw/a9fE 25 | -----END CERTIFICATE----- 26 | -------------------------------------------------------------------------------- /thor/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Simple Event-Driven IO for Python 5 | 6 | Thor is a Python library for evented IO, with a focus on enabling 7 | high-performance HTTP intermediaries. 8 | """ 9 | 10 | from __future__ import absolute_import 11 | 12 | 13 | __author__ = "Mark Nottingham " 14 | __copyright__ = """\ 15 | Copyright (c) 2005- Mark Nottingham 16 | 17 | Permission is hereby granted, free of charge, to any person obtaining a copy 18 | of this software and associated documentation files (the "Software"), to deal 19 | in the Software without restriction, including without limitation the rights 20 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 21 | copies of the Software, and to permit persons to whom the Software is 22 | furnished to do so, subject to the following conditions: 23 | 24 | The above copyright notice and this permission notice shall be included in 25 | all copies or substantial portions of the Software. 26 | 27 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 28 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 29 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 30 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 31 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 32 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 33 | THE SOFTWARE. 34 | """ 35 | __version__ = "0.12.1" 36 | 37 | from thor.loop import run, stop, time, schedule 38 | from thor.tcp import TcpClient, TcpServer 39 | from thor.udp import UdpEndpoint 40 | from thor.events import on 41 | -------------------------------------------------------------------------------- /test/test.key: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCnA6+MBfcGFldy 3 | z3lta9nHIVknEPBwSHLZjiv+Uq+d5WVoNB72DxLPTMtLGYT6KReceKVzlvPn9yjx 4 | GRXRGoGRIxItT31H8SEWn9sf2h/DPzZGMwjrcfiHqwNs1/M+D8Pz9H8aZSpI31nl 5 | goAuMzQj3B/NGMVfyjeeyyiS3X9Uq2JJ9dTYhOV6qHv71N29ulhC12XH3dLchfen 6 | ytZ8OLCfc/K5Drp06ftNJp1MP22XSZM2PVFBrJHixcB287ld+/bRfjhez5wQO2zb 7 | tTv54lsHplYa1qP7oKKvXCHw8ATplr7mnE6ngKPlZrerKjzsGrocOfyqYNTpyItT 8 | wUvlzsMRAgMBAAECggEAQkgDLiXb4C2TrPvL6/IGbrG8aPWfBmCqO87hhMAZ9Cbq 9 | 9MzikLJwu5Z1g6twC7utJCr5NoNs7t03AV/8OY00aH4ro5HNXXTDte/hDaYKPvli 10 | N7/fRVlo0aa76EFXxoSJNtMMclQ90MfbLGt7JVqq0aR1dnbjNd3R7xIfWxHOi7II 11 | UqmUyFNW21x1W8f6pAdhRzqN5gvSwA6tD1W/4mfEv6ifSTI2S1XzkUgJ9/mg2q+y 12 | 9Ai/DiCFzUuG+kb5JvmI+g+1ayCIeOLA/8+hoaOVO2L53h/M4CKmtPs9mk2rfECz 13 | EmhJHGRfDAA8KwgRMmLczR0s2aeuQOLqd5nU3eYTlQKBgQDUrFukf4ljtXC4z4Pu 14 | lfRxAqcoOOydYu3ePtUmcP3GnjT3pkd2ycGggwd0SU5frbcMH+t1+UblnzoLuIEI 15 | 8fleNXUV+VYBxLfP+0cwIeNTJ1D2blcXOhuwkojkvzQcN5S0lGMPbT3K8UN+FFl9 16 | EWonLDNj0ZN3LV++YJS2A+hbAwKBgQDJCg97AXgp26BsTvqeduYQ/2e2go2NlqLJ 17 | jkF5g3Zziw99c8EJ9IXmrB/5hSU7gDrXmn25jY5zvR6+gAGP7ngPWLbiNqTJav0u 18 | g6Vbr3i1RDSu90vqLxVmQrhh3O0dbHeWy7qTWMxlQiN6rTSSg5xBtyoZppbRjMxg 19 | TPJpBZcjWwKBgChuN+HW6RyOgsZvlOpHkbfmRDbuU3U8OHJWyZjMREJ9Ex69erqH 20 | cor8Pe+KfB7OXI0uiEneQO8oTRWrVsSHk9uoGAE8bHBboIImiAsLdjb5s3eV1HKy 21 | 9k6kHOg3vUVb/6Ywy4BESUoKgcU3Qyf2ppKZ4Y8paXpvotMDc2IC7ipXAoGBALj6 22 | PcYzUupIv/IINYdK5WNsbXQ76Z0GpmAIOWxiLsIfiLAoqszLJc1aQM1o2hpGYV3y 23 | M45gllsd/0TPKSDTDbspKJU6LCN8AtsinCqhaNJ4oHUA6+PdULZX26mICtCQ9fRV 24 | HiXnhaIB2f1Nk1rgKA07SExzwL+s6nwNMNq7HxaTAoGBAK0UuLpsQpBgk38lC+I4 25 | 9m2pNdpBclHAW6Rfq7OeKS90a/yxfU6t8Kz/B5N0Hx0iArUTsa+HKAWQ3WYdKjRw 26 | R7YTXrz+7Q7MycsQBIBSzYa1YoJQWuVBcjnBNxDpUicZIQciDkEpXvcePcgodiJa 27 | XpmTf0WOH5wtcszvz1s1LtzG 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /doc/loop.md: -------------------------------------------------------------------------------- 1 | ## Event Loops 2 | 3 | Thor creates a "default" event loop in the *thor.loop* namespace which can be 4 | run using *thor.loop.run*, and so on. If you need to run multiple loops (e.g., 5 | for testing), or want to create a loop with a custom precision, they can be 6 | explicitly created and bound to a variable using *thor.loop.make*. 7 | 8 | 9 | ### _thor.loop_ thor.loop.make ( _int_ `precision`? ) 10 | 11 | Create and return a named loop that is suitable for the current system. If 12 | `precision` is given, it indicates how often scheduled events will be run. 13 | 14 | Returned loop instances have all of the methods and instance variables that 15 | *thor.loop* has. 16 | 17 | 18 | ### _void_ thor.loop.run () 19 | 20 | Start the loop. Events can be scheduled, etc., before the loop is run, but 21 | they won't fire until it starts running. 22 | 23 | 24 | ### _void_ thor.loop.stop () 25 | 26 | Stop the loop. Some events may still run while the loop is stopping. Stopping 27 | the loop clears all scheduled events and forgets the file descriptors that 28 | it's interested in. 29 | 30 | 31 | ### _object_ thor.loop.schedule ( _int_ `delta`, _func_ `callback`, _arg_* ) 32 | 33 | Schedule callable `callback` to be called `delta` seconds from now, with 34 | one or more `arg`s. 35 | 36 | Returns an object with a *delete* () method; if called, it will remove the 37 | timeout. 38 | 39 | 40 | ### _bool_ thor.loop.running 41 | 42 | Read-only boolean that is True when the loop is running. 43 | 44 | 45 | ### _bool_ thor.loop.debug 46 | 47 | Boolean that, when True, prints warnings to STDERR when the loop is 48 | behaving oddly; e.g., a scheduled event is blocking. Default is False. 49 | 50 | 51 | ### event 'start' () 52 | 53 | Emitted right before loop starts. 54 | 55 | 56 | ### event 'stop' () 57 | 58 | Emitted right after the loop stops. 59 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | on: 4 | pull_request: 5 | # The branches below must be a subset of the branches above 6 | branches: [main] 7 | schedule: 8 | - cron: '0 19 * * 5' 9 | 10 | jobs: 11 | analyse: 12 | name: Analyse 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - name: Checkout repository 17 | uses: actions/checkout@v6 18 | with: 19 | # We must fetch at least the immediate parents so that if this is 20 | # a pull request then we can checkout the head. 21 | fetch-depth: 2 22 | 23 | # If this run was triggered by a pull request event, then checkout 24 | # the head of the pull request instead of the merge commit. 25 | - run: git checkout HEAD^2 26 | if: ${{ github.event_name == 'pull_request' }} 27 | 28 | # Initializes the CodeQL tools for scanning. 29 | - name: Initialize CodeQL 30 | uses: github/codeql-action/init@v4 31 | # Override language selection by uncommenting this and choosing your languages 32 | # with: 33 | # languages: go, javascript, csharp, python, cpp, java 34 | 35 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 36 | # If this step fails, then you should remove it and run the build manually (see below) 37 | - name: Autobuild 38 | uses: github/codeql-action/autobuild@v4 39 | 40 | # ℹ️ Command-line programs to run using the OS shell. 41 | # 📚 https://git.io/JvXDl 42 | 43 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 44 | # and modify them (or add more) to build your code if your project 45 | # uses a compiled language 46 | 47 | #- run: | 48 | # make bootstrap 49 | # make release 50 | 51 | - name: Perform CodeQL Analysis 52 | uses: github/codeql-action/analyze@v4 53 | -------------------------------------------------------------------------------- /test/test_tcp_server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import socket 4 | import sys 5 | import time 6 | import unittest 7 | 8 | import framework 9 | 10 | import thor 11 | from thor.events import on 12 | 13 | 14 | class TestTcpServer(framework.ClientServerTestCase): 15 | def create_server(self, server_side): 16 | server = thor.TcpServer(framework.test_host, 0, loop=self.loop) 17 | test_port = server.sock.getsockname()[1] 18 | server.conn_count = 0 19 | 20 | def run_server(conn): 21 | server.conn_count += 1 22 | server_side(conn) 23 | 24 | server.on("connect", run_server) 25 | 26 | def stop(): 27 | self.assertTrue(server.conn_count > 0) 28 | server.shutdown() 29 | 30 | return (stop, test_port) 31 | 32 | def create_client(self, host, port, client_side): 33 | def run_client(client_side1): 34 | client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 35 | client.connect((host, port)) 36 | client_side1(client) 37 | client.close() 38 | 39 | self.move_to_thread(target=run_client, args=(client_side,)) 40 | self.loop.schedule(1, self.loop.stop) 41 | 42 | def test_basic(self): 43 | def server_side(server_conn): 44 | self.server_recv = 0 45 | 46 | def check_data(chunk): 47 | self.assertEqual(chunk, b"foo!") 48 | self.server_recv += 1 49 | 50 | server_conn.on("data", check_data) 51 | server_conn.pause(False) 52 | server_conn.write(b"bar!") 53 | 54 | def client_side(client_conn): 55 | sent = client_conn.send(b"foo!") 56 | 57 | self.go([server_side], [client_side]) 58 | self.assertTrue(self.server_recv > 0, self.server_recv) 59 | 60 | 61 | # def test_pause(self): 62 | # def test_shutdown(self): 63 | 64 | if __name__ == "__main__": 65 | unittest.main() 66 | -------------------------------------------------------------------------------- /doc/events.md: -------------------------------------------------------------------------------- 1 | # Events 2 | 3 | ## thor.events.EventEmitter 4 | 5 | An event emitter, in the style of Node.JS. 6 | 7 | 8 | ### _void_ thor.events.EventEmitter.on ( _str_ `event`, _func_ `listener` ) 9 | 10 | Add the callable _listenter_ to the list of listeners that will be called when _event_ is emitted. 11 | 12 | 13 | ### _void_ thor.events.EventEmitter.once ( _str_ `event`, _func_ `listener` ) 14 | 15 | Call _listener_ exactly once, the next time that _event_ is emitted. 16 | 17 | 18 | ### _void_ thor.events.EventEmitter.remove_listener ( _str_ `event`, _func_ `listener` ) 19 | 20 | Remove the callable _listener_ from the list of those that will be called when _event_ is emitted. 21 | 22 | 23 | ### _void_ thor.events.EventEmitter.remove_listeners ( _str_ `event`+ ) 24 | 25 | Remove all listeners for _event_. Additional _event_s can be passed as following arguments. 26 | 27 | 28 | ### _list_ thor.events.EventEmitter.listeners ( _str_ `event` ) 29 | 30 | Return the list of callables listening for _event_. 31 | 32 | 33 | ### _void_ thor.events.EventEmitter.emit ( _str_ `event`, _arg_* ) 34 | 35 | Emit _event_ with zero or more _arg_s. 36 | 37 | 38 | ### _void_ thor.events.EventEmitter.sink ( _object_ `sink` ) 39 | 40 | Given an object _sink_, call its method (if present) that corresponds to an _events_ name if and only if there are no listeners for that event. 41 | 42 | 43 | ## Decorator thor.events.on ( _EventEmitter_ `EventEmitter`, _str_ `event` ) 44 | 45 | A decorator to nominate functions as event listeners. Its first argument is 46 | the [EventEmitter](#EventEmitter) to attach to, and the second argument is 47 | the event to listen for. 48 | 49 | For example: 50 | 51 | @on(my_event_emitter, 'blow_up') 52 | def push_red_button(thing): 53 | thing.red_button.push() 54 | 55 | If the `event` is omitted, the name of the function is used to determine 56 | the event. For example, this is equivalent to the code above: 57 | 58 | @on(my_event_emitter) 59 | def blow_up(thing): 60 | thing.red_button.push() 61 | -------------------------------------------------------------------------------- /test/test_dns.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import errno 4 | import socket 5 | import sys 6 | import threading 7 | import unittest 8 | 9 | from thor import loop 10 | from thor.dns import lookup 11 | 12 | 13 | class TestDns(unittest.TestCase): 14 | def setUp(self): 15 | self.loop = loop.make() 16 | self.loop.schedule(5, self.timeout) 17 | self.timeout_hit = False 18 | 19 | def timeout(self): 20 | self.timeout_hit = True 21 | self.loop.stop() 22 | 23 | def check_success(self, results): 24 | self.assertTrue(type(results) == list and len(results) > 0, results) 25 | 26 | def check_gai_error(self, results): 27 | self.assertTrue(isinstance(results, socket.gaierror), results) 28 | 29 | def test_basic(self): 30 | lookup(b"www.google.com", 80, socket.SOCK_STREAM, self.check_success) 31 | self.loop.run() 32 | 33 | def test_lots(self): 34 | lookup(b"www.google.com", 443, socket.SOCK_STREAM, self.check_success) 35 | lookup(b"www.facebook.com", 80, socket.SOCK_STREAM, self.check_success) 36 | lookup(b"www.example.com", 80, socket.SOCK_STREAM, self.check_success) 37 | lookup(b"www.ietf.org", 443, socket.SOCK_STREAM, self.check_success) 38 | lookup(b"www.github.com", 443, socket.SOCK_STREAM, self.check_success) 39 | lookup(b"www.twitter.com", 443, socket.SOCK_STREAM, self.check_success) 40 | lookup(b"www.abc.net.au", 80, socket.SOCK_STREAM, self.check_success) 41 | lookup(b"www.mnot.net", 443, socket.SOCK_STREAM, self.check_success) 42 | lookup(b"www.eff.org", 443, socket.SOCK_STREAM, self.check_success) 43 | lookup(b"www.aclu.org", 443, socket.SOCK_STREAM, self.check_success) 44 | lookup(b"localhost", 80, socket.SOCK_STREAM, self.check_success) 45 | self.loop.run() 46 | 47 | def test_gai(self): 48 | lookup(b"foo.foo", 23, socket.SOCK_STREAM, self.check_gai_error) 49 | lookup(b"bar.bar", 23, socket.SOCK_DGRAM, self.check_gai_error) 50 | 51 | 52 | if __name__ == "__main__": 53 | unittest.main() 54 | -------------------------------------------------------------------------------- /test/test_udp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import errno 4 | import socket 5 | import sys 6 | import threading 7 | import unittest 8 | 9 | import framework 10 | 11 | from thor import loop 12 | from thor.udp import UdpEndpoint 13 | 14 | 15 | class TestUdpEndpoint(unittest.TestCase): 16 | def setUp(self): 17 | self.loop = loop.make() 18 | self.ep1 = UdpEndpoint(self.loop) 19 | self.ep1.bind(framework.test_host, 0) 20 | self.ep1.on("datagram", self.input) 21 | self.ep1.pause(False) 22 | self.ep2 = UdpEndpoint() 23 | self.loop.schedule(5, self.timeout) 24 | self.timeout_hit = False 25 | self.datagrams = [] 26 | 27 | def tearDown(self): 28 | self.ep1.shutdown() 29 | 30 | def timeout(self): 31 | self.timeout_hit = True 32 | self.loop.stop() 33 | 34 | def input(self, data, host, port): 35 | self.datagrams.append((data, host, port)) 36 | 37 | def output(self, msg): 38 | port = self.ep1.sock.getsockname()[1] 39 | self.ep2.send(msg, framework.test_host.decode("ascii"), port) 40 | 41 | def test_basic(self): 42 | self.loop.schedule(1, self.output, b"foo!") 43 | self.loop.schedule(2, self.output, b"bar!") 44 | 45 | def check(): 46 | try: 47 | self.assertEqual(self.datagrams[0][0], b"foo!") 48 | self.assertEqual(self.datagrams[1][0], b"bar!") 49 | finally: 50 | self.loop.stop() 51 | 52 | self.loop.schedule(3, check) 53 | self.loop.run() 54 | 55 | 56 | # def test_bigdata(self): 57 | # self.loop.schedule(1, self.output, b"a" * 100) 58 | # self.loop.schedule(2, self.output, b"b" * 1000) 59 | # self.loop.schedule(3, self.output, b"c" * self.ep1.max_dgram) 60 | # 61 | # def check(): 62 | # self.assertEqual(self.datagrams[0][0], b"a" * 100) 63 | # self.assertEqual(self.datagrams[1][0], b"b" * 1000) 64 | # # we only check the first 1000 characters because, well, 65 | # # it's lossy. 66 | # self.assertEqual(self.datagrams[2][0][:1000], b"c" * 1000) 67 | # self.loop.stop() 68 | # 69 | # self.loop.schedule(4, check) 70 | # self.loop.run() 71 | 72 | 73 | # def test_pause(self): 74 | 75 | 76 | if __name__ == "__main__": 77 | unittest.main() 78 | -------------------------------------------------------------------------------- /test/test_graceful.py: -------------------------------------------------------------------------------- 1 | 2 | import unittest 3 | import time 4 | import socket 5 | import framework 6 | from thor.http import HttpServer 7 | 8 | class TestGraceful(framework.ClientServerTestCase): 9 | def create_server(self, server_side): 10 | server = HttpServer(framework.test_host, 0, loop=self.loop) 11 | assert server.tcp_server.sock is not None 12 | test_port = server.tcp_server.sock.getsockname()[1] 13 | server_side(server) 14 | 15 | def stop(): 16 | # Cleanup if needed (though loop stop handles it normally) 17 | if server.tcp_server.sock: 18 | server.tcp_server.sock.close() 19 | 20 | return (stop, test_port) 21 | 22 | def create_client(self, host, port, client_side): 23 | def run_client(client_side1): 24 | client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 25 | client.connect((host, port)) 26 | client_side1(client) 27 | client.close() 28 | 29 | self.move_to_thread(target=run_client, args=(client_side,)) 30 | 31 | def test_http_graceful(self): 32 | stop_time = 0.0 33 | start_time = time.time() 34 | 35 | def server_side(server): 36 | def trigger(): 37 | # Trigger shutdown while client is connected 38 | server.graceful_shutdown() 39 | 40 | self.loop.schedule(0.2, trigger) 41 | 42 | def on_stop(): 43 | nonlocal stop_time 44 | stop_time = time.time() 45 | self.loop.stop() 46 | 47 | server.on('stop', on_stop) 48 | 49 | def client_side(client_conn): 50 | # Connect and stay connected 51 | client_conn.send(b"GET / HTTP/1.1\r\nHost: localhost\r\n\r\n") 52 | # Sleep longer than the schedule trigger (0.2) 53 | time.sleep(1.0) 54 | client_conn.close() 55 | 56 | self.go([server_side], [client_side]) 57 | 58 | duration = stop_time - start_time 59 | # Shutdown triggered at 0.2. Client sleeps 1.0. 60 | # Stop should happen after ~1.0. 61 | # If it happened at ~0.2, it failed. 62 | self.assertTrue(duration > 0.8, f"Server stopped too early: {duration:.2f}s (expected > 0.8s)") 63 | 64 | if __name__ == "__main__": 65 | unittest.main() 66 | -------------------------------------------------------------------------------- /thor/http/client/initiate.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | import socket 3 | from typing import Callable, Union, TYPE_CHECKING 4 | 5 | from thor.dns import lookup, DnsResultList 6 | from thor.tcp import TcpClient, TcpConnection 7 | from thor.tls import TlsClient 8 | from thor.http.common import OriginType 9 | from .connection import HttpClientConnection 10 | 11 | if TYPE_CHECKING: 12 | from .client import HttpClient 13 | 14 | 15 | def initiate_connection( 16 | client: HttpClient, 17 | origin: OriginType, 18 | handle_connect: Callable[[HttpClientConnection], None], 19 | handle_error: Callable[[str, int, str], None], 20 | ) -> None: 21 | """ 22 | Creates a new TCP connection to an origin. 23 | """ 24 | attempts = 0 25 | dns_results: DnsResultList = [] 26 | 27 | def handle_dns(results: Union[DnsResultList, Exception]) -> None: 28 | nonlocal dns_results 29 | if isinstance(results, Exception): 30 | err_id = results.args[0] 31 | err_str = results.args[1] 32 | handle_error("gai", err_id, err_str) 33 | else: 34 | dns_results = results 35 | initiate_internal() 36 | 37 | def initiate_internal() -> None: 38 | nonlocal attempts 39 | dns_result = dns_results[attempts % len(dns_results)] 40 | (scheme, host, _) = origin 41 | if scheme == "http": 42 | tcp_client: Union[TcpClient, TlsClient] = TcpClient(client.loop) 43 | elif scheme == "https": 44 | tcp_client = TlsClient(client.loop) 45 | else: 46 | raise ValueError(f"unknown scheme {scheme}") 47 | tcp_client.check_ip = client.check_ip 48 | tcp_client.once("connect", handle_connect_cb) 49 | tcp_client.once("connect_error", handle_connect_error_cb) 50 | attempts += 1 51 | tcp_client.connect_dns(host.encode("idna"), dns_result, client.connect_timeout) 52 | 53 | def handle_connect_cb(tcp_conn: TcpConnection) -> None: 54 | client.conn_counts[origin] += 1 55 | conn = HttpClientConnection(client, origin, tcp_conn) 56 | handle_connect(conn) 57 | 58 | def handle_connect_error_cb(err_type: str, err_id: int, err_str: str) -> None: 59 | if err_type in ["access"]: 60 | handle_error(err_type, err_id, err_str) 61 | elif attempts > client.connect_attempts: 62 | handle_error("retry", attempts, "Too many connection attempts") 63 | else: 64 | client.loop.schedule(0, initiate_internal) 65 | 66 | (_, host, port) = origin 67 | lookup(host.encode("idna"), port, socket.SOCK_STREAM, handle_dns) 68 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "thor" 3 | dynamic = ["version"] 4 | authors = [ 5 | {name="Mark Nottingham", email="mnot@mnot.net"} 6 | ] 7 | description = "Simple Event-Driven IO for Python" 8 | readme = "README.md" 9 | requires-python = ">=3.9" 10 | license = {file = "LICENSE.md"} 11 | classifiers = [ 12 | "Intended Audience :: Developers", 13 | "License :: OSI Approved :: MIT License", 14 | "Programming Language :: Python", 15 | "Operating System :: POSIX", 16 | "Topic :: Internet :: WWW/HTTP", 17 | "Topic :: Internet :: Proxy Servers", 18 | "Topic :: Internet :: WWW/HTTP :: HTTP Servers", 19 | "Topic :: Software Development :: Libraries :: Python Modules" 20 | ] 21 | 22 | dependencies = [ 23 | "certifi", 24 | "dnspython" 25 | ] 26 | 27 | [project.optional-dependencies] 28 | dev = ["mypy", "black", "pylint", "pytest", "pytest-xdist", 29 | "py", "pytest-md", "validate-pyproject", "build"] 30 | # See https://github.com/kevlened/pytest-parallel/issues/118 31 | 32 | [project.urls] 33 | homepage = "https://github.com/mnot/thor/" 34 | 35 | [build-system] 36 | requires = [ 37 | "setuptools>=42", 38 | "wheel" 39 | ] 40 | build-backend = "setuptools.build_meta" 41 | 42 | [tool.setuptools.dynamic] 43 | version = {attr = "thor.__version__"} 44 | 45 | [tool.setuptools.packages.find] 46 | where = ["."] 47 | 48 | [tool.setuptools] 49 | include-package-data = true 50 | 51 | [tool.setuptools.package-data] 52 | SHORTNAME = ["py.typed"] 53 | 54 | [tool.mypy] 55 | follow_imports = "normal" 56 | incremental = true 57 | disallow_untyped_defs = true 58 | disallow_untyped_calls = true 59 | disallow_incomplete_defs = true 60 | warn_redundant_casts = true 61 | warn_unused_ignores = true 62 | warn_return_any = true 63 | warn_unreachable = true 64 | strict_optional = true 65 | show_error_codes = true 66 | 67 | [[tool.mypy.overrides]] 68 | module = "thor.loop" 69 | warn_unused_ignores = false 70 | 71 | 72 | [tool.pylint.basic] 73 | function-rgx = "[a-z_][a-z0-9_]{1,30}$" 74 | variable-rgx = "[a-z_][a-z0-9_]{1,30}$" 75 | attr-rgx = "[a-z_][a-z0-9_]{1,30}$" 76 | argument-rgx = "[a-z_][a-z0-9_]{1,30}$" 77 | class-attribute-rgx = "([A-Za-z_][A-Za-z0-9_]{1,30}|(__.*__))$" 78 | method-rgx = "[a-z_][a-z0-9_]{1,30}$" 79 | 80 | [tool.pylint.messages_control] 81 | disable = "C0114,C0115,C0116,W0613,W0707" 82 | 83 | [tool.pylint.reports] 84 | reports = false 85 | 86 | [tool.pylint.variables] 87 | dummy-variables-rgx = "response_start|response_body|response_done|error|formatter_done|check_done|_" 88 | 89 | [tool.pylint.design] 90 | max-args=10 91 | max-positional-arguments=8 92 | max-locals=35 93 | max-branches=25 94 | max-statements=100 95 | max-attributes=30 96 | min-public-methods=1 97 | 98 | [tool.pylint.similarities] 99 | min-similarity-lines = 10 100 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Release and Publish 2 | 3 | on: 4 | push: 5 | tags: ['v*.*.*'] 6 | 7 | jobs: 8 | build: 9 | name: Build distribution 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - uses: actions/checkout@v6 14 | - name: Set up Python 15 | uses: actions/setup-python@v6 16 | with: 17 | python-version: "3.x" 18 | - name: build 19 | run: make build 20 | - name: Store the distribution packages 21 | uses: actions/upload-artifact@v6 22 | with: 23 | name: python-package-distributions 24 | path: dist/ 25 | 26 | release: 27 | name: Release to Github 28 | needs: 29 | - build 30 | runs-on: ubuntu-latest 31 | permissions: 32 | contents: write # IMPORTANT: mandatory for making GitHub Releases 33 | steps: 34 | - uses: actions/checkout@v6 35 | with: 36 | fetch-depth: 5 37 | - name: Create a GitHub release 38 | shell: bash 39 | env: 40 | GITHUB_TOKEN: ${{ github.token }} 41 | run: | 42 | gh release create "${{ github.ref_name }}" --title "${{ github.ref_name }}" --verify-tag 43 | 44 | sign: 45 | name: >- 46 | Sign the Python distribution 47 | and upload them to GitHub Release 48 | needs: 49 | - release 50 | runs-on: ubuntu-latest 51 | 52 | permissions: 53 | contents: write # IMPORTANT: mandatory for making GitHub Releases 54 | id-token: write # IMPORTANT: mandatory for sigstore 55 | 56 | steps: 57 | - name: Download all the dists 58 | uses: actions/download-artifact@v7 59 | with: 60 | name: python-package-distributions 61 | path: dist/ 62 | - name: Sign the dists with Sigstore 63 | uses: sigstore/gh-action-sigstore-python@v3.2.0 64 | with: 65 | inputs: >- 66 | ./dist/*.tar.gz 67 | ./dist/*.whl 68 | - name: Upload artifact signatures to GitHub Release 69 | env: 70 | GITHUB_TOKEN: ${{ github.token }} 71 | # Upload to GitHub Release using the `gh` CLI. 72 | # `dist/` contains the built packages, and the 73 | # sigstore-produced signatures and certificates. 74 | run: >- 75 | gh release upload 76 | '${{ github.ref_name }}' dist/** 77 | --repo '${{ github.repository }}' 78 | 79 | publish-to-pypi: 80 | name: Publish Python distribution to PyPI 81 | needs: 82 | - sign 83 | runs-on: ubuntu-latest 84 | environment: 85 | name: pypi 86 | url: https://pypi.org/p/thor 87 | permissions: 88 | id-token: write # IMPORTANT: mandatory for trusted publishing 89 | 90 | steps: 91 | - name: Download all the dists 92 | uses: actions/download-artifact@v7 93 | with: 94 | name: python-package-distributions 95 | path: dist/ 96 | - name: Publish distribution to PyPI 97 | uses: pypa/gh-action-pypi-publish@release/v1 98 | -------------------------------------------------------------------------------- /thor/dns/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from concurrent.futures import Future, ThreadPoolExecutor 4 | from itertools import cycle, islice 5 | import socket 6 | import sys 7 | from typing import Callable, Union, Tuple, List, Iterable, Any, cast 8 | 9 | import dns.inet 10 | import dns.resolver 11 | from dns.exception import DNSException 12 | 13 | POOL_SIZE = 10 14 | 15 | Address = Union[Tuple[str, int], Tuple[str, int, int, int]] 16 | DnsResult = Tuple[ 17 | socket.AddressFamily, # pylint: disable=no-member 18 | socket.SocketKind, # pylint: disable=no-member 19 | int, 20 | str, 21 | Address, 22 | ] 23 | DnsResultList = List[DnsResult] 24 | 25 | 26 | def lookup(host: bytes, port: int, proto: int, cb: Callable[..., None]) -> None: 27 | job = _pool.submit(_lookup, host, port, proto) 28 | 29 | def done(ff: Future) -> None: 30 | cb(ff.result()) 31 | 32 | job.add_done_callback(done) 33 | 34 | 35 | def _lookup(host: bytes, port: int, socktype: int) -> Union[DnsResultList, Exception]: 36 | host_str = host.decode("idna") 37 | 38 | if dns.inet.is_address(host_str): 39 | family: socket.AddressFamily # pylint: disable=no-member 40 | if ":" in host_str: 41 | family = socket.AF_INET6 42 | else: 43 | family = socket.AF_INET 44 | return [ 45 | ( 46 | family, 47 | socket.SOCK_STREAM, 48 | socket.IPPROTO_IP, 49 | "", 50 | (host_str, port), 51 | ) 52 | ] 53 | 54 | try: 55 | results = dns.resolver.resolve_name(host_str).addresses_and_families() 56 | except DNSException as why: 57 | return socket.gaierror(1, str(why)) 58 | 59 | return _sort_dns_results( 60 | [ 61 | ( 62 | cast(socket.AddressFamily, family), # pylint: disable=no-member 63 | cast(socket.SocketKind, socktype), # pylint: disable=no-member 64 | socket.IPPROTO_IP, 65 | "", 66 | (address, port), 67 | ) 68 | for (address, family) in results 69 | ] 70 | ) 71 | 72 | 73 | def _sort_dns_results(results: DnsResultList) -> DnsResultList: 74 | ipv4results = [] 75 | ipv6results = [] 76 | for result in results: 77 | if result[0] is socket.AF_INET: 78 | ipv4results.append(result) 79 | if result[0] is socket.AF_INET6: 80 | ipv6results.append(result) 81 | return list(_roundrobin(ipv6results, ipv4results)) 82 | 83 | 84 | def _roundrobin(*iterables: Iterable) -> Iterable[Any]: 85 | "roundrobin('ABC', 'D', 'EF') --> A D E B F C" 86 | # Recipe credited to George Sakkis 87 | num_active = len(iterables) 88 | nexts = cycle(iter(it).__next__ for it in iterables) 89 | while num_active: 90 | try: 91 | for nex in nexts: 92 | yield nex() 93 | except StopIteration: 94 | # Remove the iterator we just exhausted from the cycle. 95 | num_active -= 1 96 | nexts = cycle(islice(nexts, num_active)) 97 | 98 | 99 | _pool = ThreadPoolExecutor(max_workers=POOL_SIZE) 100 | -------------------------------------------------------------------------------- /thor/http/uri.py: -------------------------------------------------------------------------------- 1 | from urllib.parse import urlsplit, urlunsplit 2 | from string import ascii_letters, digits 3 | from typing import Tuple 4 | 5 | from thor.http.error import UrlError 6 | 7 | 8 | def parse_uri(uri: bytes) -> Tuple[str, str, int, bytes, bytes]: 9 | """ 10 | Given a uri, parse out the host, port, authority and request target. 11 | Returns None if there is an error, otherwise the origin. 12 | """ 13 | try: 14 | (schemeb, authority, path, query, _) = urlsplit(uri) 15 | except UnicodeDecodeError: 16 | raise UrlError("URL has non-ascii characters") 17 | except ValueError as why: 18 | raise UrlError(why.args[0]) 19 | try: 20 | scheme = schemeb.decode("utf-8").lower() 21 | except UnicodeDecodeError: 22 | raise UrlError("URL scheme has non-ascii characters") 23 | if scheme == "http": 24 | default_port = 80 25 | elif scheme == "https": 26 | default_port = 443 27 | else: 28 | raise UrlError(f"Unsupported URL scheme '{scheme}'") 29 | if b"@" in authority: 30 | authority = authority.split(b"@", 1)[1] 31 | portb = None 32 | ipv6_literal = False 33 | if authority.startswith(b"["): 34 | ipv6_literal = True 35 | try: 36 | delimiter = authority.index(b"]") 37 | except ValueError: 38 | raise UrlError("IPv6 URL missing ]") 39 | hostb = authority[1:delimiter] 40 | rest = authority[delimiter + 1 :] 41 | if rest.startswith(b":"): 42 | portb = rest[1:] 43 | elif b":" in authority: 44 | hostb, portb = authority.rsplit(b":", 1) 45 | else: 46 | hostb = authority 47 | if portb: 48 | try: 49 | port = int(portb.decode("utf-8", "replace")) 50 | except ValueError: 51 | raise UrlError( 52 | f"Non-integer port '{portb.decode('utf-8', 'replace')}' in URL" 53 | ) 54 | if not 1 <= port <= 65535: 55 | raise UrlError(f"URL port {port} out of range") 56 | else: 57 | port = default_port 58 | try: 59 | host = hostb.decode("ascii", "strict") 60 | except UnicodeDecodeError: 61 | raise UrlError("URL host has non-ascii characters") 62 | if ipv6_literal: 63 | if not all(c in digits + ":abcdefABCDEF" for c in host): 64 | raise UrlError("URL IPv6 literal has disallowed character") 65 | else: 66 | if not all(c in ascii_letters + digits + ".-" for c in host): 67 | raise UrlError("URL hostname has disallowed character") 68 | labels = host.split(".") 69 | if any(len(l) == 0 for l in labels): 70 | raise UrlError("URL hostname has empty label") 71 | if any(len(l) > 63 for l in labels): 72 | raise UrlError("URL hostname label greater than 63 characters") 73 | # if any(l[0].isdigit() for l in labels): 74 | # self.input_error(UrlError("URL hostname label starts with digit"), False) 75 | # raise ValueError 76 | if len(host) > 255: 77 | raise UrlError("URL hostname greater than 255 characters") 78 | if path == b"": 79 | path = b"/" 80 | req_target = urlunsplit((b"", b"", path, query, b"")) 81 | return scheme, host, port, authority, req_target 82 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Thor 2 | 3 | [![CI](https://github.com/mnot/thor/actions/workflows/ci.yml/badge.svg)](https://github.com/mnot/thor/actions/workflows/ci.yml) 4 | [![Coverage Status](https://coveralls.io/repos/mnot/thor/badge.svg)](https://coveralls.io/r/mnot/thor) 5 | 6 | ## About Thor 7 | 8 | Thor is yet another [Python 3](https://python.org/) library for evented IO. 9 | 10 | There are many such libraries for Python already available. Thor focuses on making it easy to build 11 | high-performance HTTP intermediaries like proxies, load balancers, content transformation engines 12 | and service aggregators. Of course, you can use it just as a client or server too. 13 | 14 | It aims to be as fast as possible, to implement the protocols correctly, and to be simple. You can 15 | help meet these goals by contributing issues, patches and tests. 16 | 17 | Thor’s EventEmitter API is influenced by^H^H^H copied from NodeJS; if you’re familiar with Node, it 18 | shouldn’t be too hard to use Thor. However, Thor is nothing like Twisted; this is considered a 19 | feature. 20 | 21 | Currently, Thor has an event loop as well as TCP, UDP and HTTP APIs (client and server). New APIs 22 | (e.g., DNS) and capabilities should be arriving soon, along with a framework for intermediation. 23 | 24 | ## Requirements 25 | 26 | Thor just requires a current version of Python. 27 | 28 | Currently, it will run on most Posix platforms; specifically, those that offer one of `poll`, 29 | `epoll` or `kqueue`. 30 | 31 | ## Installation 32 | 33 | Using pip: 34 | 35 | > pip install thor 36 | 37 | On some operating systems, that might be `pip3`. Otherwise, download a tarball and install using: 38 | 39 | > python setup.py install 40 | 41 | ## Using Thor 42 | 43 | The [documentation](https://github.com/mnot/thor/tree/master/doc) is a good starting point; see 44 | also the docstrings for the various modules, as well as the tests, to give an idea of how to use 45 | Thor. 46 | 47 | For example, a very simple HTTP server looks like this: 48 | 49 | ```python 50 | import thor, thor.http 51 | def test_handler(exch): 52 | @thor.events.on(exch) 53 | def request_start(*args): 54 | exch.response_start(200, "OK", [('Content-Type', 'text/plain')]) 55 | exch.response_body('Hello, world!') 56 | exch.response_done([]) 57 | 58 | if __name__ == "__main__": 59 | demo_server = thor.http.HttpServer('127.0.0.1', 8000) 60 | demo_server.on('exchange', test_handler) 61 | thor.run() 62 | ``` 63 | 64 | ## Support and Contributions 65 | 66 | See [Thor's GitHub](http://github.com/mnot/thor/) to give feedback, view and [report 67 | issues](https://github.com/mnot/thor/issues), and contribute code. 68 | 69 | All helpful input is welcome, particularly code contributions via a pull request (with test cases). 70 | 71 | ## Why Thor? 72 | 73 | Thor is not only “a hammer-wielding god associated with thunder, lightning, storms, oak trees, 74 | strength, destruction, fertility, healing, and the protection of mankind”, he’s also my Norwegian 75 | Forest Cat. 76 | 77 | Thor (the software program) grew out of nbhttp, which itself came from earlier work on evented 78 | Python in [redbot](http://redbot.org/). 79 | 80 | Thor (the cat) now rules our house with a firm but benevolent paw. He gets sick if we give him any 81 | milk, though. 82 | 83 | ![Thor, the cat](https://www.mnot.net/lib/thor.jpg) 84 | -------------------------------------------------------------------------------- /thor/http/error.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Thor HTTP Errors 5 | """ 6 | 7 | from typing import Optional, Tuple 8 | 9 | 10 | class HttpError(Exception): 11 | desc = "Unknown Error" 12 | server_status: Tuple[bytes, bytes] # status this produces in a server 13 | server_recoverable = False # whether a server can recover the connection 14 | client_recoverable = False # whether a client can recover the connection 15 | 16 | def __init__(self, detail: Optional[str] = None) -> None: 17 | Exception.__init__(self) 18 | self.detail = detail 19 | 20 | def __repr__(self) -> str: 21 | status = [self.__class__.__module__ + "." + self.__class__.__name__] 22 | if self.detail: 23 | status.append(repr(self.detail)) 24 | return f"<{', '.join(status)} at {id(self):#x}>" 25 | 26 | 27 | # General parsing errors 28 | 29 | 30 | class ChunkError(HttpError): 31 | desc = "Chunked encoding error" 32 | 33 | 34 | class DuplicateCLError(HttpError): 35 | desc = "Duplicate Content-Length header" 36 | server_status = (b"400", b"Bad Request") 37 | client_recoverable = True 38 | 39 | 40 | class MalformedCLError(HttpError): 41 | desc = "Malformed Content-Length header" 42 | server_status = (b"400", b"Bad Request") 43 | 44 | 45 | class ExtraDataError(HttpError): 46 | desc = "Extra data was sent after this message was supposed to end" 47 | 48 | 49 | class OutputError(HttpError): 50 | desc = "Too many body bytes sent" 51 | 52 | 53 | class StartLineError(HttpError): 54 | desc = "The start line of the message couldn't be parsed" 55 | 56 | 57 | class HttpVersionError(HttpError): 58 | desc = "Unrecognised HTTP version" 59 | server_status = (b"505", b"HTTP Version Not Supported") 60 | 61 | 62 | class ReadTimeoutError(HttpError): 63 | desc = "Read Timeout" 64 | 65 | 66 | class TransferCodeError(HttpError): 67 | desc = "Unknown request transfer coding" 68 | server_status = (b"501", b"Not Implemented") 69 | 70 | 71 | class HeaderSpaceError(HttpError): 72 | desc = "Whitespace at the end of a header field-name" 73 | server_status = (b"400", b"Bad Request") 74 | client_recoverable = True 75 | 76 | 77 | class TopLineSpaceError(HttpError): 78 | desc = "Whitespace after top line, before first header" 79 | server_status = (b"400", b"Bad Request") 80 | client_recoverable = True 81 | 82 | 83 | class TooManyMsgsError(HttpError): 84 | desc = "Too many messages to parse" 85 | server_status = (b"400", b"Bad Request") 86 | 87 | 88 | # client-specific errors 89 | 90 | 91 | class UrlError(HttpError): 92 | desc = "Unsupported or invalid URI" 93 | server_status = (b"400", b"Bad Request") 94 | 95 | 96 | class LengthRequiredError(HttpError): 97 | desc = "Content-Length required" 98 | server_status = (b"411", b"Length Required") 99 | client_recoverable = True 100 | 101 | 102 | class DnsError(HttpError): 103 | desc = "DNS Error" 104 | server_status = (b"502", b"Bad Gateway") 105 | 106 | 107 | class ConnectError(HttpError): 108 | desc = "Connection error" 109 | server_status = (b"504", b"Gateway Timeout") 110 | 111 | 112 | class AccessError(HttpError): 113 | desc = "Access Error" 114 | server_status = (b"403", b"Forbidden") 115 | 116 | 117 | # server-specific errors 118 | 119 | 120 | class HostRequiredError(HttpError): 121 | desc = "Host header required" 122 | server_recoverable = True 123 | -------------------------------------------------------------------------------- /thor/tls.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | 4 | """ 5 | push-based asynchronous SSL/TLS-over-TCP 6 | 7 | This is a generic library for building event-based / asynchronous 8 | SSL/TLS servers and clients. 9 | """ 10 | 11 | import socket 12 | import ssl as sys_ssl 13 | from typing import Optional 14 | 15 | import certifi 16 | 17 | from thor.loop import LoopBase 18 | from thor.tcp import TcpClient, TcpConnection 19 | 20 | TcpConnection.block_errs.add(sys_ssl.SSL_ERROR_WANT_READ) 21 | TcpConnection.block_errs.add(sys_ssl.SSL_ERROR_WANT_WRITE) 22 | TcpConnection.close_errs.add(sys_ssl.SSL_ERROR_EOF) 23 | TcpConnection.close_errs.add(sys_ssl.SSL_ERROR_SSL) 24 | 25 | 26 | class TlsClient(TcpClient): 27 | """ 28 | An asynchronous SSL/TLS client. 29 | 30 | Emits: 31 | - connect (tcp_conn): upon connection 32 | - connect_error (err_type, err): if there's a problem before getting 33 | a connection. err_type is socket.error or socket.gaierror; err 34 | is the specific error encountered. 35 | 36 | To connect to a server: 37 | 38 | > c = TlsClient() 39 | > c.on('connect', conn_handler) 40 | > c.on('connect_error', error_handler) 41 | > c.connect(address) 42 | 43 | conn_handler will be called with the tcp_conn as the argument 44 | when the connection is made. 45 | """ 46 | 47 | _tls_context = sys_ssl.create_default_context(cadata=certifi.contents()) 48 | 49 | def __init__(self, loop: Optional[LoopBase] = None) -> None: 50 | TcpClient.__init__(self, loop) 51 | self.tls_sock: Optional[sys_ssl.SSLSocket] = None 52 | self._tls_context.check_hostname = False 53 | self._tls_context.verify_mode = sys_ssl.CERT_NONE 54 | 55 | def handle_connect(self) -> None: 56 | assert self.sock, "self.sock not found in handle_connect" 57 | assert self.hostname, "hostname not found in handle_connect" 58 | try: 59 | self.tls_sock = self._tls_context.wrap_socket( 60 | self.sock, 61 | do_handshake_on_connect=False, 62 | server_hostname=self.hostname.decode("idna"), 63 | ) 64 | except BlockingIOError: 65 | self.once("fd_writable", self.handle_connect) 66 | return 67 | except OSError as why: 68 | self.handle_socket_error(why, "ssl") 69 | return 70 | self.once("fd_writable", self.handshake) 71 | 72 | def handshake(self) -> None: 73 | assert self.tls_sock, "tls_sock not found in handshake" 74 | try: 75 | self.tls_sock.do_handshake() 76 | self.once("fd_writable", self.handle_tls_connect) 77 | except sys_ssl.SSLError as why: 78 | if isinstance(why, sys_ssl.SSLWantReadError): 79 | self.once("fd_writable", self.handshake) # Oh, Linux... 80 | elif isinstance(why, sys_ssl.SSLWantWriteError): 81 | self.once("fd_writable", self.handshake) 82 | else: 83 | self.handle_socket_error(why, "ssl") 84 | except socket.error as why: 85 | self.handle_socket_error(why, "ssl") 86 | except AttributeError: 87 | # For some reason, wrap_context is returning None. Try again. 88 | self.once("fd_writable", self.handshake) 89 | 90 | def handle_tls_connect(self) -> None: 91 | self.unregister_fd() 92 | if self._timeout_ev: 93 | self._timeout_ev.delete() 94 | assert self.tls_sock, "tls_sock not found in handle_tls_connect" 95 | assert self.address, "address not found in handle_tls_connect" 96 | tls_conn = TcpConnection(self.tls_sock, self.address, self.loop) 97 | self.emit("connect", tls_conn) 98 | -------------------------------------------------------------------------------- /thor/udp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | push-based asynchronous UDP 5 | 6 | This is a generic library for building event-based / asynchronous 7 | UDP servers and clients. 8 | """ 9 | 10 | import errno 11 | import socket 12 | from typing import Optional, Union 13 | 14 | from thor.dns import lookup, DnsResultList 15 | from thor.loop import EventSource, LoopBase 16 | 17 | 18 | class UdpEndpoint(EventSource): 19 | """ 20 | An asynchronous UDP endpoint. 21 | 22 | Emits: 23 | - datagram (data, address): upon recieving a datagram. 24 | 25 | To start: 26 | 27 | > s = UdpEndpoint(host, port) 28 | > s.on('datagram', datagram_handler) 29 | """ 30 | 31 | recv_buffer = 8192 32 | _block_errs = set([errno.EAGAIN, errno.EWOULDBLOCK]) 33 | 34 | def __init__(self, loop: Optional[LoopBase] = None) -> None: 35 | EventSource.__init__(self, loop) 36 | self.host: Optional[bytes] = None 37 | self.port: Optional[int] = None 38 | self._error_sent = False 39 | self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 40 | self.sock.setblocking(False) 41 | self.max_dgram = min( 42 | (2**16 - 40), self.sock.getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF) 43 | ) 44 | self.on("fd_readable", self.handle_datagram) 45 | self.register_fd(self.sock.fileno()) 46 | 47 | def __repr__(self) -> str: 48 | status = [self.__class__.__module__ + "." + self.__class__.__name__] 49 | return f"<{', '.join(status)} at {id(self):#x}>" 50 | 51 | def bind(self, host: bytes, port: int) -> None: 52 | """ 53 | Bind the socket bound to host:port. If called, must be before 54 | sending or receiving. 55 | 56 | Can raise socket.error if binding fails. 57 | """ 58 | self.host = host 59 | self.port = port 60 | self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 61 | lookup(host, port, socket.SOCK_DGRAM, self._continue_bind) 62 | 63 | def _continue_bind(self, dns_results: Union[DnsResultList, Exception]) -> None: 64 | if isinstance(dns_results, Exception): 65 | self.handle_socket_error(dns_results, "gai") 66 | return 67 | self.sock.bind(dns_results[0][4]) 68 | 69 | def shutdown(self) -> None: 70 | "Close the listening socket." 71 | self.remove_listeners("fd_readable") 72 | self.sock.close() 73 | 74 | def pause(self, paused: bool) -> None: 75 | "Control incoming datagram events." 76 | if paused: 77 | self.event_del("fd_readable") 78 | else: 79 | self.event_add("fd_readable") 80 | 81 | def send(self, datagram: bytes, host: str, port: int) -> None: 82 | "send datagram to host:port." 83 | try: 84 | self.sock.sendto(datagram, (host, port)) 85 | except socket.error as why: 86 | if why in self._block_errs: 87 | pass # we drop these on the floor. It's UDP, after all. 88 | else: 89 | raise 90 | 91 | def handle_datagram(self) -> None: 92 | "Handle an incoming datagram, emitting the 'datagram' event." 93 | while True: 94 | try: 95 | data, addr = self.sock.recvfrom(self.recv_buffer) 96 | except socket.error as why: 97 | if why.args[0] in self._block_errs: 98 | break 99 | raise 100 | self.emit("datagram", data, addr[0], addr[1]) 101 | 102 | def handle_socket_error(self, why: Exception, err_type: str = "socket") -> None: 103 | err_id = why.args[0] 104 | err_str = why.args[1] 105 | if self._error_sent: 106 | return 107 | self._error_sent = True 108 | self.unregister_fd() 109 | self.emit("socket_error", err_type, err_id, err_str) 110 | if self.sock: 111 | self.sock.close() 112 | -------------------------------------------------------------------------------- /doc/tcp.md: -------------------------------------------------------------------------------- 1 | # TCP 2 | 3 | 4 | ## thor.TcpClient ( _[thor.loop](loop.md)_ `loop`? ) 5 | 6 | A TCP client. If `loop` is omitted, the "default" loop will be used. 7 | 8 | Note that new connections will not emit *data* events until they are unpaused; see [thor.tcp.TcpConnection.pause](#void-thortcptcpconnnectionpause--bool-paused-). 9 | 10 | For example: 11 | 12 | import sys 13 | import thor 14 | 15 | test_host, test_port = sys.argv[1:2] 16 | 17 | def handle_connect(conn): 18 | conn.on('data', sys.stdout.write) 19 | conn.on('close', thor.stop) 20 | conn.write(b"GET /\n\n") 21 | conn.pause(False) 22 | 23 | def handle_err(err_type, err): 24 | sys.stderr.write(str(err_type)) 25 | thor.stop() 26 | 27 | c = thor.TcpClient() 28 | c.on('connect', handle_connect) 29 | c.on('connect_error', handle_err) 30 | c.connect(test_host, test_port) 31 | thor.run() 32 | 33 | 34 | ### _void_ thor.TcpClient.connect ( _str_ `host`, _int_ `port`, _int_ `timeout`? ) 35 | 36 | Call to initiate a connection to `port` on `host`. [connect](#event-connect--tcpconnection-connection-) will be emitted when a connection is available, and [connect_error](#event-connect_error--errtype-error-) will be emitted when it fails. 37 | 38 | If `timeout` is given, it specifies a connect timeout, in seconds. If the timeout is exceeded and no connection or explicit failure is encountered, [connect_error](#event-connect_error--errtype-error-) will be emitted with *socket.error* as the _errtype_ and *errno.ETIMEDOUT* as the _error_. 39 | 40 | 41 | #### event 'connect' ( _[TcpConnection](#thortcptcpconnection)_ `connection` ) 42 | 43 | Emitted when the connection has succeeded. 44 | 45 | 46 | #### event 'connect\_error' ( _errtype_, _error_ ) 47 | 48 | Emitted when the connection failed. _errtype_ is *socket.error* or *socket.gaierror*; _error_ is the error type specific to the type. 49 | 50 | 51 | ## thor.TcpServer ( _str_ `host`, _int_ `port`, _[thor.loop](loop.md)_ `loop`? ) 52 | 53 | A TCP server. `host` and `port` specify the host and port to listen on, respectively; if given, `loop` specifies the *thor.loop* to use. If `loop` is omitted, the "default" loop will be used. 54 | 55 | Note that new connections will not emit *data* events until they are unpaused; see [pause](#void-thortcptcpconnnectionpause--bool-paused-). 56 | 57 | For example: 58 | 59 | s = TcpServer("localhost", 8000) 60 | s.on('connect', handle_conn) 61 | 62 | 63 | ### event 'start' () 64 | 65 | Emitted when the server starts. 66 | 67 | 68 | ### event 'connect' ( _[TcpConnection](#thortcptcpconnection)_ `connection` ) 69 | 70 | Emitted when a new connection is accepted by the server. 71 | 72 | 73 | ### event 'stop' () 74 | 75 | Emitted when the server stops. 76 | 77 | 78 | ### _void_ thor.TcpServer.shutdown () 79 | 80 | Stops the server from accepting new connections. Existing connections will remain active. 81 | 82 | 83 | ### _void_ thor.TcpServer.graceful_shutdown () 84 | 85 | Stops the server from accepting new connections, and waits for all active connections to close before emitting [stop](#event-stop-). 86 | 87 | 88 | 89 | ## thor.tcp.TcpConnection 90 | 91 | A single TCP connection. 92 | 93 | 94 | ### event 'data' ( _bytes_ `data` ) 95 | 96 | Emitted when incoming _data_ is received by the connection. See [thor.tcp.TcpConnection.pause](#void-thortcptcpconnnectionpause--bool-paused-) to control these events. 97 | 98 | 99 | ### event 'close' () 100 | 101 | Emitted when the connection is closed, either because the other side has closed it, or because of a network problem. 102 | 103 | 104 | ### event 'pause' ( _bool_ `paused` ) 105 | 106 | Emitted to indicate the pause state, using `paused`, of the outgoing side of the connection (i.e., the *write* side). 107 | 108 | When True, the connection buffers are full, and *write* should not be called again until this event is emitted again with `paused` as False. 109 | 110 | 111 | ### _void_ thor.tcp.TcpConnection.write ( _bytes_ `data` ) 112 | 113 | Write _data_ to the connection. Note that it may not be sent immediately. 114 | 115 | 116 | ### _void_ thor.tcp.TcpConnnection.pause ( _bool_ `paused` ) 117 | 118 | Controls the incoming side of the connection (i.e., *data* events). When `paused` is True, incoming [data](#event-data--bytes-data-) events will stop; when `paused` is false, they will resume again. 119 | 120 | Note that by default, *TcpConnection*s are paused; i.e., to read from them, you must first *thor.tcp.TcpConnection.pause*(_False_). 121 | 122 | 123 | ### _void_ thor.tcp.TcpConnection.close () 124 | 125 | Close the connection. If there is data still in the outgoing buffer, it will be written before the socket is shut down. 126 | -------------------------------------------------------------------------------- /Makefile.pyproject: -------------------------------------------------------------------------------- 1 | # 2 | # Common Makefile for Python projects 3 | # 4 | # 5 | # Insert `include Makefile.pyproject` at the bottom of your Makefile to enable these 6 | # rules. Requires Makefile.venv to also be present. 7 | # 8 | # Requires $PROJECT to be set to the name of the project, and for that to be the source folder. 9 | # 10 | # This Makefile provides the following targets: 11 | # 12 | # clean_py 13 | # Clean Python-related temporary files 14 | # tidy_py 15 | # Run black on Python files in $PROJECT 16 | # lint_py 17 | # Run pylint on $PROJECT, and validate pyproject.toml 18 | # typecheck_py 19 | # Run mypy on $PROJECT 20 | # bump-calver 21 | # Bump the __version__ in $PROJECT/__init__.py using calver (https://calver.org/) 22 | # bump-semver[-major,-minor,-micro] 23 | # Bump the __version__ in $PROJECT/__init__.py using semver (https://semver.org/) 24 | # version <<< COMMITS TO GIT 25 | # Bump the version according to $VERSIONING (default: semver-micro). 26 | # build 27 | # Build the project 28 | # pypi-pub 29 | # Publish the project to pypi 30 | # release <<< TAGS AND PUSHES TO GITHUB 31 | # Release the project on Pypi and Github releases (requires /.github/workflows/release.yml) 32 | # 33 | # See also Makefile.venv. 34 | # 35 | # 36 | # Make sure the following are in requirements.txt: 37 | # - pylint 38 | # - mypy 39 | # - black 40 | # - build 41 | # - validate-pyproject 42 | 43 | PY?=python 44 | VERSIONING?=semver-micro 45 | YEAR=`date +%Y` 46 | MONTH=`date +%m` 47 | 48 | # Python-specific targets 49 | 50 | .PHONY: clean_py 51 | clean_py: clean-venv 52 | find . -d -type d -name __pycache__ -exec rm -rf {} \; 53 | rm -rf build dist MANIFEST $(PROJECT).egg-info .mypy_cache *.log changelog.md 54 | 55 | .PHONY: tidy_py 56 | tidy_py: venv 57 | $(VENV)/black $(PROJECT) 58 | 59 | .PHONY: lint_py 60 | lint_py: venv 61 | PYTHONPATH=$(VENV) $(VENV)/pylint --output-format=colorized $(PROJECT) 62 | $(VENV)/validate-pyproject pyproject.toml 63 | 64 | .PHONY: typecheck_py 65 | typecheck_py: venv 66 | PYTHONPATH=$(VENV) $(VENV)/python -m mypy $(PROJECT) 67 | 68 | ## Release 69 | 70 | .PHONY: py_version 71 | py_version: venv 72 | $(eval VERSION=$(shell $(VENV)/python -c "import $(PROJECT); print($(PROJECT).__version__)")) 73 | $(eval VER_MAJOR=$(shell echo $(VERSION) | cut -d. -f1)) 74 | $(eval VER_MINOR=$(shell echo $(VERSION) | cut -d. -f2)) 75 | $(eval VER_MICRO=$(shell echo $(VERSION) | cut -d. -f3)) 76 | ## for calendar-based versioning 77 | $(eval NEXT_CALMICRO=$(shell \ 78 | if [[ $(YEAR) != $(VER_MAJOR) || $(MONTH) != $(VER_MINOR) ]] ; then \ 79 | echo "1"; \ 80 | else \ 81 | echo $$(( $(VER_MICRO) + 1 )); \ 82 | fi; \ 83 | )) 84 | ## for semantic versioning 85 | $(eval NEXT_SEMMAJOR=$(shell \ 86 | echo $$(( $(VER_MAJOR) + 1 )); \ 87 | )) 88 | $(eval NEXT_SEMMINOR=$(shell \ 89 | echo $$(( $(VER_MINOR) + 1 )); \ 90 | )) 91 | $(eval NEXT_SEMMICRO=$(shell \ 92 | echo $$(( $(VER_MICRO) + 1 )); \ 93 | )) 94 | 95 | .PHONY: bump-calver 96 | bump-calver: py_version 97 | sed -i "" -e "s/$(VERSION)/$(YEAR).$(MONTH).$(NEXT_CALMICRO)/" $(PROJECT)/__init__.py 98 | 99 | .PHONY: bump-semver-micro 100 | bump-semver-micro: py_version 101 | sed -i "" -e "s/$(VERSION)/$(VER_MAJOR).$(VER_MINOR).$(NEXT_SEMMICRO)/" $(PROJECT)/__init__.py 102 | 103 | .PHONY: bump-semver-minor 104 | bump-semver-minor: py_version 105 | sed -i "" -e "s/$(VERSION)/$(VER_MAJOR).$(NEXT_SEMMINOR).0/" $(PROJECT)/__init__.py 106 | 107 | .PHONY: bump-semver-major 108 | bump-semver-major: py_version 109 | sed -i "" -e "s/$(VERSION)/$(NEXT_SEMMAJOR).0.0/" $(PROJECT)/__init__.py 110 | 111 | .PHONY: version 112 | version: typecheck lint test bump-$(VERSIONING) 113 | git add $(PROJECT)/__init__.py 114 | git commit -m 'bump version' 115 | 116 | .PHONY: build 117 | build: venv 118 | $(VENV)/python -m build 119 | 120 | changelog.md: 121 | $(eval PREV_RELEASE=$(shell git tag --sort=-taggerdate -l v* | head -1)) 122 | git --no-pager log --pretty="- %s" $(PREV_RELEASE)..HEAD --grep "^Changed:" --grep "^Fixed:" --grep "^Added:" --grep "^Changed:" --grep "^Removed:" --output=$@ 123 | sed -i "" -e "1s/^/\n\n/" $@ 124 | 125 | # Manual pypi publication; no GitHub release 126 | .PHONY: pypi-pub 127 | pypi-pub: venv build 128 | $(VENV)/python -m twine upload dist/* 129 | 130 | # Github release with Pypi publication; requires /.github/workflows/release.yml 131 | .PHONY: release 132 | release: py_version changelog.md 133 | git tag -a "v$(VERSION)" -F changelog.md 134 | rm -f changelog.md 135 | git push 136 | git push --tags origin # github action will push to pypi and create a release 137 | 138 | 139 | include Makefile.venv 140 | 141 | ## Patch venv to install from pyproject.toml 142 | 143 | venv: | $(VENV)/$(MARKER)-dev 144 | $(VENV)/$(MARKER)-dev: $(VENV)/$(MARKER) 145 | $(VENV)/pip install -e .[dev] 146 | touch $@ 147 | -------------------------------------------------------------------------------- /thor/events.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Event utilities, including: 5 | 6 | * EventEmitter - in the style of Node.JS. 7 | * on - a decorator for making functions and methods listen to events. 8 | """ 9 | 10 | import contextvars 11 | from collections import defaultdict 12 | from typing import Optional, Any, Callable, Dict, List 13 | 14 | 15 | class EventEmitter: 16 | """ 17 | An event emitter, in the style of Node.JS. 18 | """ 19 | 20 | def __init__(self) -> None: 21 | self.__events: Dict[str, List[Callable]] = defaultdict(list) 22 | self.__sink: object = None 23 | 24 | def __getstate__(self) -> Dict[str, Any]: 25 | state = self.__dict__.copy() 26 | try: 27 | del state["_EventEmitter__events"] 28 | except KeyError: 29 | pass 30 | return state 31 | 32 | def on(self, event: str, listener: Callable) -> None: 33 | """ 34 | Call listener when event is emitted. 35 | """ 36 | if isinstance(listener, ContextWrapper): 37 | wrapped = listener 38 | else: 39 | wrapped = ContextWrapper(listener) 40 | self.__events[event].append(wrapped) 41 | self.emit("newListener", event, wrapped) 42 | 43 | def once(self, event: str, listener: Callable) -> None: 44 | """ 45 | Call listener the first time event is emitted. 46 | """ 47 | 48 | def mycall(*args: Any) -> None: 49 | self.remove_listener(event, mycall) 50 | listener(*args) 51 | 52 | mycall.__name__ = getattr(listener, "__name__", "listener") 53 | self.on(event, mycall) 54 | 55 | def remove_listener(self, event: str, listener: Callable) -> None: 56 | """ 57 | Remove a specific listener from an event. 58 | 59 | If called for a specific listener by a previous listener 60 | for the same event, that listener will not be fired. 61 | """ 62 | self.__events.get(event, [listener]).remove(listener) 63 | 64 | def remove_listeners(self, *events: str) -> None: 65 | """ 66 | Remove all listeners from an event; if no event 67 | is specified, remove all listeners for all events. 68 | 69 | If called from an event listener, other listeners 70 | for that event will still be fired. 71 | """ 72 | if events: 73 | for event in events: 74 | self.__events[event] = [] 75 | else: 76 | self.__events = defaultdict(list) 77 | 78 | def listeners(self, event: str) -> List[Callable]: 79 | """ 80 | Return a list of listeners for an event. 81 | """ 82 | return self.__events.get(event, []) 83 | 84 | def events(self) -> List[str]: 85 | """ 86 | Return a list of events being listened for. 87 | """ 88 | return list(self.__events) 89 | 90 | def emit(self, event: str, *args: Any) -> None: 91 | """ 92 | Emit the event (with any given args) to 93 | its listeners. 94 | """ 95 | events = self.__events.get(event, []) 96 | if events: 97 | for ev in events: 98 | ev(*args) 99 | else: 100 | sink_event = getattr(self.__sink, event, None) 101 | if sink_event: 102 | sink_event(*args) 103 | 104 | def sink(self, sink: object) -> None: 105 | """ 106 | If no listeners are found for an event, call 107 | the method that shares the event's name (if present) 108 | on the event sink. 109 | """ 110 | self.__sink = sink 111 | 112 | 113 | class ContextWrapper: 114 | """ 115 | A wrapper for a listener that captures the current context and runs 116 | the listener within it. 117 | """ 118 | 119 | def __init__(self, listener: Callable) -> None: 120 | self.listener = listener 121 | self.context = contextvars.copy_context() 122 | self.__name__ = getattr(listener, "__name__", "listener") 123 | 124 | def __call__(self, *args: Any, **lwargs: Any) -> Any: 125 | return self.context.run(self.listener, *args, **lwargs) 126 | 127 | def __eq__(self, other: Any) -> bool: 128 | if isinstance(other, ContextWrapper): 129 | return self.listener == other.listener 130 | return bool(self.listener == other) 131 | 132 | def __hash__(self) -> int: 133 | return hash(self.listener) 134 | 135 | 136 | def on(obj: EventEmitter, event: Optional[str] = None) -> Callable: 137 | """ 138 | Decorator to call a function when an object emits 139 | the specified event. 140 | """ 141 | 142 | def wrap(funk: Callable) -> Callable: 143 | name = getattr(funk, "__name__", "listener") 144 | obj.on(event or name, funk) 145 | return funk 146 | 147 | return wrap 148 | -------------------------------------------------------------------------------- /test/test_events.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import unittest 5 | 6 | from thor.events import EventEmitter, on 7 | 8 | 9 | class TestEventEmitter(unittest.TestCase): 10 | def setUp(self): 11 | class Thing(EventEmitter): 12 | def __init__(self): 13 | EventEmitter.__init__(self) 14 | self.foo_count = 0 15 | self.bar_count = 0 16 | self.rem1_count = 0 17 | self.rem2_count = 0 18 | self.on("foo", self.handle_foo) 19 | self.once("bar", self.handle_bar) 20 | self.on("baz", self.handle_baz) 21 | self.on("rem1", self.handle_rem1) 22 | self.on("rem1", self.handle_rem1a) 23 | self.on("rem2", self.handle_rem2) 24 | self.on("rem2", self.handle_rem2a) 25 | 26 | def handle_foo(self): 27 | self.foo_count += 1 28 | 29 | def handle_bar(self): 30 | self.bar_count += 1 31 | 32 | def handle_baz(self): 33 | raise Exception("Baz wasn't removed.") 34 | 35 | def handle_rem1(self): 36 | self.rem1_count += 1 37 | self.remove_listeners() 38 | self.emit("foo") 39 | 40 | def handle_rem1a(self): 41 | self.rem1_count += 1 42 | 43 | def handle_rem2(self): 44 | self.rem2_count += 1 45 | self.remove_listener("rem2", self.handle_rem2a) 46 | 47 | def handle_rem2a(self): 48 | self.rem2_count += 1 49 | 50 | self.t = Thing() 51 | 52 | def test_basic(self): 53 | self.assertEqual(self.t.foo_count, 0) 54 | self.t.emit("foo") 55 | self.assertEqual(self.t.foo_count, 1) 56 | self.t.emit("foo") 57 | self.assertEqual(self.t.foo_count, 2) 58 | 59 | def test_once(self): 60 | self.assertEqual(self.t.bar_count, 0) 61 | self.t.emit("bar") 62 | self.assertEqual(self.t.bar_count, 1) 63 | self.t.emit("bar") 64 | self.assertEqual(self.t.bar_count, 1) 65 | 66 | def test_remove_listener(self): 67 | self.t.remove_listener("foo", self.t.handle_foo) 68 | self.t.emit("foo") 69 | self.assertEqual(self.t.foo_count, 0) 70 | 71 | def test_remove_listeners_named(self): 72 | self.t.remove_listeners("baz") 73 | self.t.emit("baz") 74 | 75 | def test_remove_listeners_named_multiple(self): 76 | self.t.remove_listeners("baz", "foo") 77 | self.t.emit("baz") 78 | self.t.emit("foo") 79 | self.assertEqual(self.t.foo_count, 0) 80 | 81 | def test_remove_listeners_all(self): 82 | self.t.emit("foo") 83 | self.t.remove_listeners() 84 | self.t.emit("foo") 85 | self.assertEqual(self.t.foo_count, 1) 86 | self.t.emit("baz") 87 | 88 | def test_sink(self): 89 | class TestSink: 90 | def __init__(self): 91 | self.bam_count = 0 92 | 93 | def bam(self): 94 | self.bam_count += 1 95 | 96 | s = TestSink() 97 | self.t.sink(s) 98 | self.assertEqual(s.bam_count, 0) 99 | self.t.emit("bam") 100 | self.assertEqual(s.bam_count, 1) 101 | self.assertEqual(self.t.foo_count, 0) 102 | self.t.emit("foo") 103 | self.assertEqual(self.t.foo_count, 1) 104 | 105 | def test_on_named(self): 106 | self.t.boom_count = 0 107 | 108 | @on(self.t, "boom") 109 | def do(): 110 | self.t.boom_count += 1 111 | 112 | self.assertEqual(self.t.boom_count, 0) 113 | self.t.emit("boom") 114 | self.assertEqual(self.t.boom_count, 1) 115 | 116 | def test_on_default(self): 117 | self.t.boom_count = 0 118 | 119 | @on(self.t) 120 | def boom(): 121 | self.t.boom_count += 1 122 | 123 | self.assertEqual(self.t.boom_count, 0) 124 | self.t.emit("boom") 125 | self.assertEqual(self.t.boom_count, 1) 126 | 127 | def test_remove_listeners_recursion(self): 128 | """ 129 | All event listeners are called for a given 130 | event, even if one of the previous listeners 131 | calls remove_listeners(). 132 | """ 133 | self.assertEqual(self.t.rem1_count, 0) 134 | self.t.emit("rem1") 135 | self.assertEqual(self.t.foo_count, 0) 136 | self.assertEqual(self.t.rem1_count, 2) 137 | 138 | def test_remove_listener_recursion(self): 139 | """ 140 | Removing a later listener specifically for 141 | a given event causes it not to be run. 142 | """ 143 | self.assertEqual(self.t.rem2_count, 0) 144 | self.t.emit("rem2") 145 | self.assertEqual(self.t.rem2_count, 1) 146 | 147 | 148 | if __name__ == "__main__": 149 | unittest.main() 150 | -------------------------------------------------------------------------------- /test/test_tls_client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | try: 4 | import SocketServer 5 | except ImportError: 6 | import socketserver as SocketServer 7 | 8 | import errno 9 | import socket 10 | import ssl 11 | import sys 12 | import threading 13 | import unittest 14 | 15 | import framework 16 | 17 | from thor import loop 18 | from thor.tls import TlsClient 19 | import pytest 20 | 21 | 22 | class LittleTlsServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): 23 | def __init__( 24 | self, 25 | server_address, 26 | RequestHandlerClass, 27 | certfile, 28 | keyfile, 29 | ssl_version=ssl.PROTOCOL_TLS_SERVER, 30 | bind_and_activate=True, 31 | ): 32 | self.context = ssl.SSLContext(ssl_version) 33 | self.context.load_cert_chain(certfile, keyfile) 34 | 35 | SocketServer.TCPServer.__init__( 36 | self, server_address, RequestHandlerClass, bind_and_activate 37 | ) 38 | 39 | def get_request(self): 40 | newsocket, fromaddr = self.socket.accept() 41 | connstream = self.context.wrap_socket(newsocket, server_side=True) 42 | return connstream, fromaddr 43 | 44 | 45 | class TestTlsClientConnect(framework.ClientServerTestCase): 46 | def setUp(self): 47 | self.loop = loop.make() 48 | self.connect_count = 0 49 | self.error_count = 0 50 | self.last_error_type = None 51 | self.last_error = None 52 | self.last_error_str = None 53 | self.timeout_hit = False 54 | self.conn = None 55 | 56 | def check_connect(conn): 57 | self.conn = conn 58 | self.assertTrue(conn.tcp_connected) 59 | self.connect_count += 1 60 | conn.write(b"GET / HTTP/1.0\r\n\r\n") 61 | self.loop.schedule(1, self.loop.stop) 62 | 63 | def check_error(err_type, err_id, err_str): 64 | self.error_count += 1 65 | self.last_error_type = err_type 66 | self.last_error = err_id 67 | self.last_error_str = err_str 68 | self.loop.schedule(1, self.loop.stop) 69 | 70 | def timeout(): 71 | self.loop.stop() 72 | self.timeout_hit = True 73 | 74 | self.timeout = timeout 75 | self.client = TlsClient(self.loop) 76 | self.client.on("connect", check_connect) 77 | self.client.on("connect_error", check_error) 78 | 79 | def start_server(self): 80 | self.server = LittleTlsServer( 81 | (framework.tls_host, framework.tls_port), 82 | framework.LittleRequestHandler, 83 | "test/test.cert", 84 | "test/test.key", 85 | ) 86 | 87 | def serve(): 88 | self.server.serve_forever(poll_interval=0.1) 89 | 90 | self.move_to_thread(serve) 91 | 92 | def stop_server(self): 93 | self.server.shutdown() 94 | self.server.server_close() 95 | 96 | def test_connect(self): 97 | self.start_server() 98 | self.client.connect(framework.tls_host, framework.tls_port) 99 | self.loop.schedule(5, self.timeout) 100 | try: 101 | self.loop.run() 102 | finally: 103 | self.stop_server() 104 | self.assertEqual( 105 | self.error_count, 106 | 0, 107 | (self.last_error_type, self.last_error, self.last_error_str), 108 | ) 109 | self.assertEqual(self.timeout_hit, False) 110 | self.assertEqual(self.connect_count, 1) 111 | 112 | def test_connect_refused(self): 113 | self.client.connect(framework.refuse_host, framework.refuse_port) 114 | self.loop.schedule(3, self.timeout) 115 | self.loop.run() 116 | self.assertEqual(self.connect_count, 0) 117 | self.assertEqual(self.error_count, 1) 118 | self.assertEqual(self.last_error, errno.ECONNREFUSED) 119 | self.assertEqual(self.timeout_hit, False) 120 | 121 | def test_connect_noname(self): 122 | self.client.connect(b"does.not.exist.invalid", framework.tls_port) 123 | self.loop.schedule(3, self.timeout) 124 | self.loop.run() 125 | self.assertEqual(self.connect_count, 0) 126 | self.assertEqual(self.error_count, 1) 127 | self.assertEqual(self.last_error_type, "socket") 128 | self.assertEqual(self.last_error, socket.EAI_NONAME) 129 | self.assertEqual(self.timeout_hit, False) 130 | 131 | def test_connect_timeout(self): 132 | self.client.connect(framework.timeout_host, framework.timeout_port, 1) 133 | self.loop.schedule(3, self.timeout) 134 | self.loop.run() 135 | self.assertEqual(self.connect_count, 0) 136 | self.assertEqual(self.error_count, 1) 137 | self.assertEqual(self.last_error_type, "socket") 138 | self.assertEqual(self.last_error, errno.ETIMEDOUT) 139 | self.assertEqual(self.timeout_hit, False) 140 | 141 | 142 | # def test_pause(self): 143 | 144 | if __name__ == "__main__": 145 | unittest.main() 146 | -------------------------------------------------------------------------------- /test/framework.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Framework for testing clients and servers, moving one of them into 5 | a separate thread. 6 | """ 7 | 8 | try: 9 | import SocketServer 10 | except ImportError: 11 | import socketserver as SocketServer 12 | 13 | import os 14 | import socket 15 | import sys 16 | import threading 17 | import time 18 | import unittest 19 | 20 | import thor 21 | from thor.http.common import HttpMessageHandler, States 22 | 23 | test_host = b"127.0.0.1" 24 | tls_host = test_host 25 | tls_port = 24443 26 | timeout_host = b"www.mnot.net" 27 | timeout_port = 31000 28 | refuse_host = test_host 29 | refuse_port = 45000 30 | timeout = 30 31 | 32 | 33 | class ClientServerTestCase(unittest.TestCase): 34 | def setUp(self): 35 | self.loop = thor.loop.make() 36 | self.loop.debug = False 37 | self.timeout_hit = False 38 | 39 | def tearDown(self): 40 | if self.loop.running: 41 | sys.stdout.write("WARNING: loop still running at end of test.") 42 | self.loop.stop() 43 | 44 | @staticmethod 45 | def move_to_thread(target, args=None): 46 | t = threading.Thread(target=target, args=args or []) 47 | t.daemon = True 48 | t.start() 49 | 50 | def go(self, server_sides, client_sides, timeout=timeout): 51 | """ 52 | Start the server(s), handling connections with server_side (handler), 53 | and then run the client(s), calling client_side (client). 54 | 55 | One of the handlers MUST stop the loop before the timeout, which 56 | is considered failure. 57 | """ 58 | 59 | assert len(server_sides) == len(client_sides) 60 | steps = [] 61 | for server_side in server_sides: 62 | steps.append(self.create_server(server_side)) 63 | i = 0 64 | for client_side in client_sides: 65 | self.create_client(test_host, steps[i][1], client_side) 66 | i += 1 67 | 68 | def do_timeout(): 69 | self.loop.stop() 70 | self.timeout_hit = True 71 | 72 | self.loop.schedule(timeout, do_timeout) 73 | try: 74 | self.loop.run() 75 | finally: 76 | [step[0]() for step in steps] 77 | self.assertEqual(self.timeout_hit, False) 78 | 79 | def create_server(self, server_side): 80 | raise NotImplementedError 81 | 82 | def create_client(self, host, port, client_side): 83 | raise NotImplementedError 84 | 85 | 86 | class DummyHttpParser(HttpMessageHandler): 87 | default_state = States.WAITING 88 | 89 | def __init__(self, *args, **kw): 90 | HttpMessageHandler.__init__(self, *args, **kw) 91 | self.test_top_line = None 92 | self.test_hdrs = None 93 | self.test_body = b"" 94 | self.test_trailers = None 95 | self.test_err = None 96 | self.test_states = [] 97 | 98 | def input_start( 99 | self, top_line, hdr_tuples, conn_tokens, transfer_codes, content_length 100 | ): 101 | self.test_states.append("START") 102 | self.test_top_line = top_line 103 | self.test_hdrs = hdr_tuples 104 | return True, True 105 | 106 | def input_body(self, chunk): 107 | self.test_states.append("BODY") 108 | self.test_body += chunk 109 | 110 | def input_end(self, trailers): 111 | self.test_states.append("END") 112 | self.test_trailers = trailers 113 | 114 | def input_error(self, err): 115 | self.test_states.append("ERROR") 116 | self.test_err = err 117 | return False # never recover. 118 | 119 | def output(self, data): 120 | pass 121 | 122 | def output_done(self): 123 | pass 124 | 125 | def check(self, asserter, expected): 126 | """ 127 | Check the parsed message against expected attributes and 128 | assert using asserter as necessary. 129 | """ 130 | aE = asserter.assertEqual 131 | aE(expected.get("top_line", self.test_top_line), self.test_top_line) 132 | aE(expected.get("hdrs", self.test_hdrs), self.test_hdrs) 133 | aE(expected.get("body", self.test_body), self.test_body) 134 | aE(expected.get("trailers", self.test_trailers), self.test_trailers) 135 | aE(expected.get("error", self.test_err), self.test_err) 136 | aE(expected.get("states", self.test_states), self.test_states) 137 | 138 | 139 | class LittleRequestHandler(SocketServer.BaseRequestHandler): 140 | def handle(self): 141 | # Echo back to the client 142 | data = self.request.recv(1024) 143 | self.request.send(data) 144 | self.request.close() 145 | 146 | 147 | def make_fifo(filename): 148 | try: 149 | os.unlink(filename) 150 | except OSError: 151 | pass # wasn't there 152 | try: 153 | os.mkfifo(filename) 154 | except OSError as e: 155 | print(f"Failed to create FIFO: {e}") 156 | else: 157 | r = os.open(filename, os.O_RDONLY | os.O_NONBLOCK) 158 | w = os.open(filename, os.O_WRONLY | os.O_NONBLOCK) 159 | return r, w 160 | -------------------------------------------------------------------------------- /thor/http/client/client.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | from collections import defaultdict 3 | import socket 4 | from typing import Optional, Callable, List, Dict, Tuple 5 | 6 | import thor 7 | from thor.loop import LoopBase 8 | from thor.http.common import OriginType 9 | 10 | from .initiate import initiate_connection 11 | from .exchange import HttpClientExchange 12 | from .connection import HttpClientConnection 13 | 14 | 15 | class HttpClient: 16 | "An asynchronous HTTP client." 17 | 18 | def __init__(self, loop: Optional[LoopBase] = None) -> None: 19 | self.loop = loop or thor.loop._loop 20 | self.idle_timeout: int = 60 # seconds 21 | self.connect_attempts: int = 3 22 | self.connect_timeout: int = 3 # seconds 23 | self.read_timeout: Optional[int] = None # seconds 24 | self.retry_limit: int = 2 25 | self.retry_delay: float = 0.5 # seconds 26 | self.max_server_conn: int = 6 27 | self.check_ip: Optional[Callable[[str], bool]] = None 28 | self.careful: bool = True 29 | self._idle_conns: Dict[OriginType, List[HttpClientConnection]] = defaultdict( 30 | list 31 | ) 32 | self.conn_counts: Dict[OriginType, int] = defaultdict(int) 33 | self._req_q: Dict[OriginType, List[Tuple[Callable, Callable]]] = defaultdict( 34 | list 35 | ) 36 | self.loop.once("stop", self._close_conns) 37 | 38 | def exchange(self) -> HttpClientExchange: 39 | return HttpClientExchange(self) 40 | 41 | def attach_conn( 42 | self, 43 | origin: OriginType, 44 | handle_connect: Callable[[HttpClientConnection], None], 45 | handle_connect_error: Callable, 46 | ) -> None: 47 | "Find an idle connection for origin, or create a new one." 48 | while True: 49 | try: 50 | conn = self._idle_conns[origin].pop() 51 | if not conn.tcp_connected: 52 | self._conn_is_dead(conn) 53 | continue 54 | except IndexError: # No idle conns available. 55 | if origin in self._idle_conns and not self._idle_conns[origin]: 56 | del self._idle_conns[origin] 57 | self._new_conn(origin, handle_connect, handle_connect_error) 58 | break 59 | if conn.tcp_connected: 60 | if conn.idler: 61 | conn.idler.delete() 62 | handle_connect(conn) 63 | break 64 | 65 | def release_conn(self, conn: HttpClientConnection) -> None: 66 | "Add an idle connection back to the pool." 67 | conn.detach() 68 | if not conn.tcp_connected: 69 | self._conn_is_dead(conn) 70 | return 71 | 72 | origin = conn.origin 73 | assert origin, "origin not found in release_conn" 74 | 75 | def idle_close() -> None: 76 | "Remove the connection from the pool when it closes." 77 | if conn.idler: 78 | conn.idler.delete() 79 | self._conn_is_dead(conn) 80 | try: 81 | self._idle_conns[origin].remove(conn) 82 | if not self._idle_conns[origin]: 83 | del self._idle_conns[origin] 84 | except (KeyError, ValueError): 85 | pass 86 | 87 | if self._req_q[origin]: 88 | handle_connect = self._req_q[origin].pop(0)[0] 89 | handle_connect(conn) 90 | elif self.idle_timeout > 0: 91 | conn.once("close", idle_close) 92 | conn.idler = self.loop.schedule(self.idle_timeout, idle_close) 93 | self._idle_conns[origin].append(conn) 94 | else: 95 | conn.close() 96 | self._conn_is_dead(conn) 97 | 98 | def dead_conn(self, conn: HttpClientConnection) -> None: 99 | "Notify the client that a connection is dead." 100 | conn.detach() 101 | if conn.tcp_connected: 102 | conn.close() 103 | self._conn_is_dead(conn) 104 | 105 | def _conn_is_dead(self, conn: HttpClientConnection) -> None: 106 | origin = conn.origin 107 | assert origin, "origin not found in _conn_is_dead" 108 | self.conn_counts[origin] -= 1 109 | if self.conn_counts[origin] <= 0: 110 | if origin in self.conn_counts: 111 | del self.conn_counts[origin] 112 | if self._req_q[origin]: 113 | (handle_connect, handle_connect_error) = self._req_q[origin].pop(0) 114 | self._new_conn(origin, handle_connect, handle_connect_error) 115 | 116 | def _new_conn( 117 | self, 118 | origin: OriginType, 119 | handle_connect: Callable[[HttpClientConnection], None], 120 | handle_error: Callable[[str, int, str], None], 121 | ) -> None: 122 | "Create a new connection." 123 | if self.conn_counts[origin] >= self.max_server_conn: 124 | self._req_q[origin].append((handle_connect, handle_error)) 125 | return 126 | initiate_connection(self, origin, handle_connect, handle_error) 127 | 128 | def _close_conns(self) -> None: 129 | "Close all idle HTTP connections." 130 | for conn_list in list(self._idle_conns.values()): 131 | for conn in list(conn_list): 132 | try: 133 | conn.close() 134 | except socket.error: 135 | pass 136 | self._idle_conns.clear() 137 | -------------------------------------------------------------------------------- /test/test_loop.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import errno 4 | import os 5 | import socket 6 | import sys 7 | import tempfile 8 | import time as systime 9 | import unittest 10 | 11 | from framework import make_fifo 12 | 13 | import thor.loop 14 | 15 | 16 | class IOStopper(thor.loop.EventSource): 17 | def __init__(self, testcase, loop): 18 | thor.loop.EventSource.__init__(self, loop) 19 | self.testcase = testcase 20 | self.r_fd, self.w_fd = make_fifo(f"tmp_fifo_{testcase.id()}") 21 | self.on("fd_writable", self.write) 22 | self.register_fd(self.w_fd, "fd_writable") 23 | 24 | def write(self): 25 | self.testcase.assertTrue(self.loop.running) 26 | self.loop.stop() 27 | os.close(self.r_fd) 28 | os.close(self.w_fd) 29 | os.unlink(f"tmp_fifo_{self.testcase.id()}") 30 | 31 | 32 | class TestLoop(unittest.TestCase): 33 | def setUp(self): 34 | self.loop = thor.loop.make() 35 | self.i = 0 36 | 37 | def increment_counter(self): 38 | self.i += 1 39 | 40 | def test_start_event(self): 41 | self.loop.on("start", self.increment_counter) 42 | self.loop.schedule(1, self.loop.stop) 43 | self.loop.run() 44 | self.assertEqual(self.i, 1) 45 | 46 | def test_stop_event(self): 47 | self.loop.on("stop", self.increment_counter) 48 | self.loop.schedule(1, self.loop.stop) 49 | self.loop.run() 50 | self.assertEqual(self.i, 1) 51 | 52 | def test_run(self): 53 | def check_running(): 54 | self.assertTrue(self.loop.running) 55 | 56 | self.loop.schedule(0, check_running) 57 | self.loop.schedule(1, self.loop.stop) 58 | self.loop.run() 59 | 60 | def test_scheduled_stop(self): 61 | self.loop.schedule(1, self.loop.stop) 62 | self.loop.run() 63 | self.assertFalse(self.loop.running) 64 | 65 | def test_io_stop(self): 66 | r = IOStopper(self, self.loop) 67 | self.loop.run() 68 | self.assertFalse(self.loop.running) 69 | 70 | def test_run_stop_run(self): 71 | def check_running(): 72 | self.assertTrue(self.loop.running) 73 | 74 | self.loop.schedule(0, check_running) 75 | self.loop.schedule(1, self.loop.stop) 76 | self.loop.run() 77 | self.assertFalse(self.loop.running) 78 | self.loop.schedule(0, check_running) 79 | self.loop.schedule(1, self.loop.stop) 80 | self.loop.run() 81 | 82 | def test_schedule(self): 83 | run_time = 3 # how long to run for 84 | 85 | def check_time(start_time): 86 | now = systime.time() 87 | self.assertTrue( 88 | now - run_time - start_time <= self.loop.precision, 89 | "now: %s run_time: %s start_time: %s precision: %s" 90 | % (now, run_time, start_time, self.loop.precision), 91 | ) 92 | self.loop.stop() 93 | 94 | self.loop.schedule(run_time, check_time, systime.time()) 95 | self.loop.run() 96 | 97 | def test_schedule_delete(self): 98 | def not_good(): 99 | assert Exception, "this event should not have happened." 100 | 101 | e = self.loop.schedule(2, not_good) 102 | self.loop.schedule(1, e.delete) 103 | self.loop.schedule(3, self.loop.stop) 104 | self.loop.run() 105 | 106 | def test_time(self): 107 | run_time = 2 108 | 109 | def check_time(): 110 | self.assertTrue( 111 | abs(systime.time() - self.loop.time()) <= self.loop.precision 112 | ) 113 | self.loop.stop() 114 | 115 | self.loop.schedule(run_time, check_time) 116 | self.loop.run() 117 | 118 | 119 | class TestEventSource(unittest.TestCase): 120 | def setUp(self): 121 | self.loop = thor.loop.make() 122 | self.es = thor.loop.EventSource(self.loop) 123 | self.events_seen = [] 124 | self.r_fd, self.w_fd = make_fifo(f"tmp_fifo_{self.id}") 125 | 126 | def tearDown(self): 127 | os.close(self.r_fd) 128 | os.close(self.w_fd) 129 | os.unlink(f"tmp_fifo_{self.id}") 130 | 131 | def test_EventSource_register(self): 132 | self.es.register_fd(self.r_fd) 133 | self.assertTrue(self.r_fd in list(self.loop._fd_targets)) 134 | 135 | def test_EventSource_unregister(self): 136 | self.es.register_fd(self.r_fd) 137 | self.assertTrue(self.r_fd in list(self.loop._fd_targets)) 138 | self.es.unregister_fd() 139 | self.assertFalse(self.r_fd in list(self.loop._fd_targets)) 140 | 141 | def test_EventSource_event_del(self): 142 | self.es.register_fd(self.r_fd, "fd_readable") 143 | self.es.on("fd_readable", self.readable_check) 144 | self.es.event_del("fd_readable") 145 | os.write(self.w_fd, b"foo") 146 | self.loop._run_fd_events() 147 | self.assertFalse("fd_readable" in self.events_seen) 148 | 149 | def test_EventSource_readable(self): 150 | self.es.register_fd(self.r_fd, "fd_readable") 151 | self.es.on("fd_readable", self.readable_check) 152 | os.write(self.w_fd, b"foo") 153 | self.loop._run_fd_events() 154 | self.assertTrue("fd_readable" in self.events_seen) 155 | 156 | def test_EventSource_not_readable(self): 157 | self.es.register_fd(self.r_fd, "fd_readable") 158 | self.es.on("fd_readable", self.readable_check) 159 | self.loop._run_fd_events() 160 | self.assertFalse("fd_readable" in self.events_seen) 161 | 162 | def readable_check(self, check=b"foo"): 163 | data = os.read(self.r_fd, 5) 164 | self.assertEqual(data, check) 165 | self.events_seen.append("fd_readable") 166 | 167 | def close_check(self): 168 | self.events_seen.append("fd_close") 169 | 170 | 171 | if __name__ == "__main__": 172 | unittest.main() 173 | -------------------------------------------------------------------------------- /test/test_tcp_client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | try: 4 | import SocketServer 5 | except ImportError: 6 | import socketserver as SocketServer 7 | 8 | import errno 9 | import socket 10 | import unittest 11 | 12 | import framework 13 | 14 | from thor import loop 15 | from thor.tcp import TcpClient, TcpConnection 16 | 17 | 18 | class LittleServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): 19 | allow_reuse_address = True 20 | 21 | 22 | class TestTcpClientConnect(framework.ClientServerTestCase): 23 | def setUp(self): 24 | self.loop = loop.make() 25 | self.connect_count = 0 26 | self.error_count = 0 27 | self.last_error_type = None 28 | self.last_error = None 29 | self.timeout_hit = False 30 | self.conn = None 31 | 32 | def check_connect(conn): 33 | self.conn = conn 34 | self.assertTrue(conn.tcp_connected) 35 | self.connect_count += 1 36 | conn.write(b"test") 37 | self.loop.schedule(1, self.loop.stop) 38 | 39 | def check_error(err_type, err_id, err_str): 40 | self.error_count += 1 41 | self.last_error_type = err_type 42 | self.last_error = err_id 43 | self.last_error_str = err_str 44 | self.loop.schedule(1, self.loop.stop) 45 | 46 | def timeout(): 47 | self.loop.stop() 48 | self.timeout_hit = True 49 | 50 | self.timeout = timeout 51 | self.client = TcpClient(self.loop) 52 | self.client.on("connect", check_connect) 53 | self.client.on("connect_error", check_error) 54 | 55 | def start_server(self): 56 | self.server = LittleServer( 57 | (framework.test_host, 0), framework.LittleRequestHandler 58 | ) 59 | test_port = self.server.server_address[1] 60 | 61 | def serve(): 62 | self.server.serve_forever(poll_interval=0.1) 63 | 64 | self.move_to_thread(serve) 65 | return test_port 66 | 67 | def stop_server(self): 68 | self.server.shutdown() 69 | self.server.server_close() 70 | 71 | def test_connect(self): 72 | test_port = self.start_server() 73 | self.client.connect(framework.test_host, test_port) 74 | self.loop.schedule(2, self.timeout) 75 | try: 76 | self.loop.run() 77 | finally: 78 | self.stop_server() 79 | self.assertEqual(self.connect_count, 1) 80 | self.assertEqual(self.error_count, 0) 81 | self.assertEqual(self.timeout_hit, False) 82 | 83 | def test_connect_refused(self): 84 | self.client.connect(framework.refuse_host, framework.refuse_port) 85 | self.loop.schedule(3, self.timeout) 86 | self.loop.run() 87 | self.assertEqual(self.connect_count, 0) 88 | self.assertEqual(self.error_count, 1) 89 | self.assertEqual(self.last_error_type, "socket") 90 | self.assertEqual(self.last_error, errno.ECONNREFUSED) 91 | self.assertEqual(self.timeout_hit, False) 92 | 93 | def test_connect_noname(self): 94 | self.client.connect(b"does.not.exist.invalid", 80) 95 | self.loop.schedule(3, self.timeout) 96 | self.loop.run() 97 | self.assertEqual(self.connect_count, 0) 98 | self.assertEqual(self.error_count, 1) 99 | self.assertEqual(self.last_error_type, "socket", self.last_error_str) 100 | self.assertEqual(self.last_error, socket.EAI_NONAME) 101 | self.assertEqual(self.timeout_hit, False) 102 | 103 | def test_ip_check(self): 104 | test_port = self.start_server() 105 | 106 | def ip_check(dns_result): 107 | return False 108 | 109 | self.client.check_ip = ip_check 110 | self.client.connect(framework.test_host, test_port) 111 | self.loop.schedule(2, self.timeout) 112 | try: 113 | self.loop.run() 114 | finally: 115 | self.stop_server() 116 | self.assertEqual(self.connect_count, 0) 117 | self.assertEqual(self.error_count, 1) 118 | self.assertEqual(self.timeout_hit, False) 119 | 120 | def test_connect_timeout(self): 121 | self.client.connect(framework.timeout_host, framework.timeout_port, 1) 122 | self.loop.schedule(3, self.timeout) 123 | self.loop.run() 124 | self.assertEqual(self.connect_count, 0) 125 | self.assertEqual(self.error_count, 1) 126 | self.assertEqual(self.last_error_type, "socket") 127 | self.assertEqual( 128 | self.last_error, 129 | errno.ETIMEDOUT, 130 | errno.errorcode.get(self.last_error, self.last_error), 131 | ) 132 | self.assertEqual(self.timeout_hit, False) 133 | 134 | 135 | # def test_pause(self): 136 | 137 | 138 | def test_write_closed_crash(self): 139 | # Use real sockets to avoid issues with loop registration 140 | rsock, wsock = socket.socketpair() 141 | rsock.setblocking(False) 142 | 143 | # Create connection and close it 144 | conn = TcpConnection(rsock, ("127.0.0.1", 80), self.loop) 145 | conn._close() # Force internal close 146 | wsock.close() 147 | 148 | # Expectation: write should raise OSError 149 | with self.assertRaisesRegex(OSError, "Connection closed"): 150 | conn.write(b"foo") 151 | 152 | def test_stuck_close(self): 153 | rsock, wsock = socket.socketpair() 154 | rsock.setblocking(False) 155 | 156 | conn = TcpConnection(rsock, ("127.0.0.1", 80), self.loop) 157 | # Simulate buffered data 158 | conn._write_buffer.append(b"pending") 159 | 160 | # Call close 161 | conn.close() 162 | 163 | # Expectation: conn.tcp_connected should still be True because it's waiting to flush 164 | self.assertTrue(conn.tcp_connected) 165 | self.assertTrue(conn._closing) 166 | 167 | # Use abort to force close 168 | conn.abort() 169 | self.assertFalse(conn.tcp_connected) 170 | wsock.close() 171 | 172 | if __name__ == "__main__": 173 | unittest.main() 174 | -------------------------------------------------------------------------------- /thor/http/client/connection.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | from typing import Optional, List, Tuple, TYPE_CHECKING 3 | 4 | from thor.events import EventEmitter 5 | from thor.tcp import TcpConnection 6 | from thor.http.common import ( 7 | HttpMessageHandler, 8 | States, 9 | Delimiters, 10 | RawHeaderListType, 11 | OriginType, 12 | no_body_status, 13 | ) 14 | from thor.http import error 15 | from thor.http.error import ( 16 | StartLineError, 17 | HttpVersionError, 18 | ) 19 | from thor.loop import ScheduledEvent 20 | 21 | if TYPE_CHECKING: 22 | from .client import HttpClient 23 | from .exchange import HttpClientExchange 24 | 25 | 26 | class HttpClientConnection(HttpMessageHandler, EventEmitter): 27 | """ 28 | A persistent connection for the HTTP client. 29 | """ 30 | 31 | default_state = States.WAITING 32 | 33 | def __init__( 34 | self, client: HttpClient, origin: OriginType, tcp_conn: TcpConnection 35 | ) -> None: 36 | self.client = client 37 | self.origin = origin 38 | self.tcp_conn = tcp_conn 39 | HttpMessageHandler.__init__(self) 40 | EventEmitter.__init__(self) 41 | self.active_exchange: Optional[HttpClientExchange] = None 42 | self.last_active_exchange: Optional[HttpClientExchange] = None 43 | self.res_version: Optional[bytes] = None 44 | self.reusable = False 45 | self.idler: Optional[ScheduledEvent] = None 46 | self._read_timeout_ev: Optional[ScheduledEvent] = None 47 | 48 | self.tcp_conn.on("data", self.handle_input) 49 | self.tcp_conn.once("close", self._conn_closed) 50 | self.tcp_conn.on("pause", self._conn_paused) 51 | 52 | def attach(self, exchange: HttpClientExchange) -> None: 53 | self.active_exchange = exchange 54 | self.last_active_exchange = None 55 | self.careful = exchange.careful 56 | self.reusable = False 57 | self._input_state = self.default_state 58 | self._input_delimit = Delimiters.NONE 59 | self._input_body_left = 0 60 | self._output_state = States.WAITING 61 | self._output_delimit = Delimiters.NONE 62 | self.tcp_conn.pause(False) 63 | if self._input_buffer: 64 | self.handle_input(b"") 65 | self.set_timeout(self.client.connect_timeout, "connect") 66 | 67 | def detach(self) -> None: 68 | self.last_active_exchange = self.active_exchange 69 | self.active_exchange = None 70 | 71 | def close(self) -> None: 72 | self.tcp_conn.close() 73 | 74 | def kill(self) -> None: 75 | if self.tcp_connected: 76 | self.close() 77 | self.client.dead_conn(self) 78 | 79 | def set_timeout(self, timeout: float, kind: str) -> None: 80 | self.clear_timeout() 81 | self._read_timeout_ev = self.client.loop.schedule( 82 | timeout, self.input_error, error.ReadTimeoutError(kind) 83 | ) 84 | 85 | def clear_timeout(self) -> None: 86 | if self._read_timeout_ev: 87 | self._read_timeout_ev.delete() 88 | self._read_timeout_ev = None 89 | 90 | @property 91 | def tcp_connected(self) -> bool: 92 | return self.tcp_conn.tcp_connected 93 | 94 | def _conn_closed(self) -> None: 95 | if self._input_buffer: 96 | self.handle_input(b"") 97 | self.clear_timeout() 98 | self.client.dead_conn(self) 99 | self.emit("close") 100 | if self.active_exchange: 101 | self.active_exchange.conn_closed(self._input_state, self._input_delimit) 102 | elif self.last_active_exchange: 103 | self.last_active_exchange.conn_closed( 104 | self._input_state, self._input_delimit 105 | ) 106 | 107 | def _conn_paused(self, paused: bool) -> None: 108 | self.emit("pause", paused) 109 | if self.active_exchange: 110 | self.active_exchange.req_body_pause(paused) 111 | 112 | # HttpMessageHandler overrides 113 | 114 | def handle_input(self, inbytes: bytes) -> None: 115 | if self.active_exchange or not self.reusable: 116 | HttpMessageHandler.handle_input(self, inbytes) 117 | else: 118 | self._input_buffer.append(inbytes) 119 | 120 | # HttpMessageHandler implementation 121 | 122 | def input_start( 123 | self, 124 | top_line: bytes, 125 | hdr_tuples: RawHeaderListType, 126 | conn_tokens: List[bytes], 127 | transfer_codes: List[bytes], 128 | content_length: Optional[int], 129 | ) -> Tuple[bool, bool]: 130 | try: 131 | proto_version, status_txt = top_line.split(None, 1) 132 | proto, self.res_version = proto_version.rsplit(b"/", 1) 133 | except (ValueError, IndexError): 134 | self.input_error(StartLineError(top_line.decode("utf-8", "replace"))) 135 | raise ValueError 136 | if proto != b"HTTP" or self.res_version not in [b"1.0", b"1.1"]: 137 | self.input_error(HttpVersionError(proto_version.decode("utf-8", "replace"))) 138 | raise ValueError 139 | 140 | try: 141 | res_code, res_phrase = status_txt.split(None, 1) 142 | except ValueError: 143 | res_code = status_txt.rstrip() 144 | res_phrase = b"" 145 | 146 | if b"close" not in conn_tokens: 147 | if ( 148 | self.res_version == b"1.0" and b"keep-alive" in conn_tokens 149 | ) or self.res_version in [b"1.1"]: 150 | self.reusable = True 151 | 152 | if self.reusable: 153 | self.default_state = States.WAITING 154 | else: 155 | self.default_state = States.QUIET 156 | 157 | is_final = not res_code.startswith(b"1") 158 | allows_body = is_final and (res_code not in no_body_status) 159 | 160 | if self.active_exchange: 161 | if self.active_exchange.method == b"HEAD": 162 | allows_body = False 163 | 164 | if is_final: 165 | self.active_exchange.res_version = self.res_version 166 | self.active_exchange.emit( 167 | "response_start", res_code, res_phrase, hdr_tuples 168 | ) 169 | else: 170 | self.active_exchange.emit( 171 | "response_nonfinal", res_code, res_phrase, hdr_tuples 172 | ) 173 | 174 | return allows_body, is_final 175 | 176 | def input_body(self, chunk: bytes) -> None: 177 | if self.active_exchange: 178 | self.active_exchange.emit("response_body", chunk) 179 | 180 | def input_end(self, trailers: RawHeaderListType) -> None: 181 | self.clear_timeout() 182 | exchange = self.active_exchange 183 | if self.reusable: 184 | self.client.release_conn(self) 185 | else: 186 | self.client.dead_conn(self) 187 | if exchange: 188 | exchange.input_end_notify(trailers) 189 | 190 | def input_error(self, err: error.HttpError) -> None: 191 | self.clear_timeout() 192 | exchange = self.active_exchange or self.last_active_exchange 193 | self.client.dead_conn(self) 194 | if exchange: 195 | exchange.input_error(err) 196 | 197 | def output(self, data: bytes) -> None: 198 | self.tcp_conn.write(data) 199 | 200 | def output_done(self) -> None: 201 | pass 202 | -------------------------------------------------------------------------------- /Makefile.venv: -------------------------------------------------------------------------------- 1 | # 2 | # SEAMLESSLY MANAGE PYTHON VIRTUAL ENVIRONMENT WITH A MAKEFILE 3 | # 4 | # https://github.com/sio/Makefile.venv v2023.04.17 5 | # 6 | # 7 | # Insert `include Makefile.venv` at the bottom of your Makefile to enable these 8 | # rules. 9 | # 10 | # When writing your Makefile use '$(VENV)/python' to refer to the Python 11 | # interpreter within virtual environment and '$(VENV)/executablename' for any 12 | # other executable in venv. 13 | # 14 | # This Makefile provides the following targets: 15 | # venv 16 | # Use this as a dependency for any target that requires virtual 17 | # environment to be created and configured 18 | # python, ipython 19 | # Use these to launch interactive Python shell within virtual environment 20 | # shell, bash, zsh 21 | # Launch interactive command line shell. "shell" target launches the 22 | # default shell Makefile executes its rules in (usually /bin/sh). 23 | # "bash" and "zsh" can be used to refer to the specific desired shell. 24 | # show-venv 25 | # Show versions of Python and pip, and the path to the virtual environment 26 | # clean-venv 27 | # Remove virtual environment 28 | # $(VENV)/executable_name 29 | # Install `executable_name` with pip. Only packages with names matching 30 | # the name of the corresponding executable are supported. 31 | # Use this as a lightweight mechanism for development dependencies 32 | # tracking. E.g. for one-off tools that are not required in every 33 | # developer's environment, therefore are not included into 34 | # requirements.txt or setup.py. 35 | # Note: 36 | # Rules using such target or dependency MUST be defined below 37 | # `include` directive to make use of correct $(VENV) value. 38 | # Example: 39 | # codestyle: $(VENV)/pyflakes 40 | # $(VENV)/pyflakes . 41 | # See `ipython` target below for another example. 42 | # 43 | # This Makefile can be configured via following variables: 44 | # PY 45 | # Command name for system Python interpreter. It is used only initially to 46 | # create the virtual environment 47 | # Default: python3 48 | # REQUIREMENTS_TXT 49 | # Space separated list of paths to requirements.txt files. 50 | # Paths are resolved relative to current working directory. 51 | # Default: requirements.txt 52 | # 53 | # Non-existent files are treated as hard dependencies, 54 | # recipes for creating such files must be provided by the main Makefile. 55 | # Providing empty value (REQUIREMENTS_TXT=) turns off processing of 56 | # requirements.txt even when the file exists. 57 | # SETUP_PY, SETUP_CFG, PYPROJECT_TOML, VENV_LOCAL_PACKAGE 58 | # Space separated list of paths to files that contain build instructions 59 | # for local Python packages. Corresponding packages will be installed 60 | # into venv in editable mode along with all their dependencies. 61 | # Default: setup.py setup.cfg pyproject.toml (whichever present) 62 | # 63 | # Non-existent and empty values are treated in the same way as for REQUIREMENTS_TXT. 64 | # WORKDIR 65 | # Parent directory for the virtual environment. 66 | # Default: current working directory. 67 | # VENVDIR 68 | # Python virtual environment directory. 69 | # Default: $(WORKDIR)/.venv 70 | # 71 | # This Makefile was written for GNU Make and may not work with other make 72 | # implementations. 73 | # 74 | # 75 | # Copyright (c) 2019-2023 Vitaly Potyarkin 76 | # 77 | # Licensed under the Apache License, Version 2.0 78 | # 79 | # 80 | 81 | 82 | # 83 | # Configuration variables 84 | # 85 | 86 | WORKDIR?=. 87 | VENVDIR?=$(WORKDIR)/.venv 88 | REQUIREMENTS_TXT?=$(wildcard requirements.txt) # Multiple paths are supported (space separated) 89 | SETUP_PY?=$(wildcard setup.py) # Multiple paths are supported (space separated) 90 | SETUP_CFG?=$(foreach s,$(SETUP_PY),$(wildcard $(patsubst %setup.py,%setup.cfg,$(s)))) 91 | PYPROJECT_TOML?=$(wildcard pyproject.toml) 92 | VENV_LOCAL_PACKAGE?=$(SETUP_PY) $(SETUP_CFG) $(PYPROJECT_TOML) 93 | MARKER=.initialized-with-Makefile.venv 94 | 95 | 96 | # 97 | # Python interpreter detection 98 | # 99 | 100 | _PY_AUTODETECT_MSG=Detected Python interpreter: $(PY). Use PY environment variable to override 101 | 102 | ifeq (ok,$(shell test -e /dev/null 2>&1 && echo ok)) 103 | NULL_STDERR=2>/dev/null 104 | else 105 | NULL_STDERR=2>NUL 106 | endif 107 | 108 | ifndef PY 109 | _PY_OPTION:=python3 110 | ifeq (ok,$(shell $(_PY_OPTION) -c "print('ok')" $(NULL_STDERR))) 111 | PY=$(_PY_OPTION) 112 | endif 113 | endif 114 | 115 | ifndef PY 116 | _PY_OPTION:=$(VENVDIR)/bin/python 117 | ifeq (ok,$(shell $(_PY_OPTION) -c "print('ok')" $(NULL_STDERR))) 118 | PY=$(_PY_OPTION) 119 | $(info $(_PY_AUTODETECT_MSG)) 120 | endif 121 | endif 122 | 123 | ifndef PY 124 | _PY_OPTION:=$(subst /,\,$(VENVDIR)/Scripts/python) 125 | ifeq (ok,$(shell $(_PY_OPTION) -c "print('ok')" $(NULL_STDERR))) 126 | PY=$(_PY_OPTION) 127 | $(info $(_PY_AUTODETECT_MSG)) 128 | endif 129 | endif 130 | 131 | ifndef PY 132 | _PY_OPTION:=py -3 133 | ifeq (ok,$(shell $(_PY_OPTION) -c "print('ok')" $(NULL_STDERR))) 134 | PY=$(_PY_OPTION) 135 | $(info $(_PY_AUTODETECT_MSG)) 136 | endif 137 | endif 138 | 139 | ifndef PY 140 | _PY_OPTION:=python 141 | ifeq (ok,$(shell $(_PY_OPTION) -c "print('ok')" $(NULL_STDERR))) 142 | PY=$(_PY_OPTION) 143 | $(info $(_PY_AUTODETECT_MSG)) 144 | endif 145 | endif 146 | 147 | ifndef PY 148 | define _PY_AUTODETECT_ERR 149 | Could not detect Python interpreter automatically. 150 | Please specify path to interpreter via PY environment variable. 151 | endef 152 | $(error $(_PY_AUTODETECT_ERR)) 153 | endif 154 | 155 | 156 | # 157 | # Internal variable resolution 158 | # 159 | 160 | VENV=$(VENVDIR)/bin 161 | EXE= 162 | # Detect windows 163 | ifeq (win32,$(shell $(PY) -c "import __future__, sys; print(sys.platform)")) 164 | VENV=$(VENVDIR)/Scripts 165 | EXE=.exe 166 | endif 167 | 168 | touch=touch $(1) 169 | ifeq (,$(shell command -v touch $(NULL_STDERR))) 170 | # https://ss64.com/nt/touch.html 171 | touch=type nul >> $(subst /,\,$(1)) && copy /y /b $(subst /,\,$(1))+,, $(subst /,\,$(1)) 172 | endif 173 | 174 | RM?=rm -f 175 | ifeq (,$(shell command -v $(firstword $(RM)) $(NULL_STDERR))) 176 | RMDIR:=rd /s /q 177 | else 178 | RMDIR:=$(RM) -r 179 | endif 180 | 181 | 182 | # 183 | # Virtual environment 184 | # 185 | 186 | .PHONY: venv 187 | venv: $(VENV)/$(MARKER) 188 | 189 | .PHONY: clean-venv 190 | clean-venv: 191 | -$(RMDIR) "$(VENVDIR)" 192 | 193 | .PHONY: show-venv 194 | show-venv: venv 195 | @$(VENV)/python -c "import sys; print('Python ' + sys.version.replace('\n',''))" 196 | @$(VENV)/pip --version 197 | @echo venv: $(VENVDIR) 198 | 199 | .PHONY: debug-venv 200 | debug-venv: 201 | @echo "PATH (Shell)=$$PATH" 202 | @$(MAKE) --version 203 | $(info PATH (GNU Make)="$(PATH)") 204 | $(info SHELL="$(SHELL)") 205 | $(info PY="$(PY)") 206 | $(info REQUIREMENTS_TXT="$(REQUIREMENTS_TXT)") 207 | $(info VENV_LOCAL_PACKAGE="$(VENV_LOCAL_PACKAGE)") 208 | $(info VENVDIR="$(VENVDIR)") 209 | $(info VENVDEPENDS="$(VENVDEPENDS)") 210 | $(info WORKDIR="$(WORKDIR)") 211 | 212 | 213 | # 214 | # Dependencies 215 | # 216 | 217 | ifneq ($(strip $(REQUIREMENTS_TXT)),) 218 | VENVDEPENDS+=$(REQUIREMENTS_TXT) 219 | endif 220 | 221 | ifneq ($(strip $(VENV_LOCAL_PACKAGE)),) 222 | VENVDEPENDS+=$(VENV_LOCAL_PACKAGE) 223 | endif 224 | 225 | $(VENV): 226 | $(PY) -m venv $(VENVDIR) 227 | $(VENV)/python -m pip install --upgrade pip setuptools wheel 228 | 229 | $(VENV)/$(MARKER): $(VENVDEPENDS) | $(VENV) 230 | ifneq ($(strip $(REQUIREMENTS_TXT)),) 231 | $(VENV)/pip install $(foreach path,$(REQUIREMENTS_TXT),-r $(path)) 232 | endif 233 | ifneq ($(strip $(VENV_LOCAL_PACKAGE)),) 234 | $(VENV)/pip install $(foreach path,$(sort $(VENV_LOCAL_PACKAGE)),-e $(dir $(path))) 235 | endif 236 | $(call touch,$(VENV)/$(MARKER)) 237 | 238 | 239 | # 240 | # Interactive shells 241 | # 242 | 243 | .PHONY: python 244 | python: venv 245 | exec $(VENV)/python 246 | 247 | .PHONY: ipython 248 | ipython: $(VENV)/ipython 249 | exec $(VENV)/ipython 250 | 251 | .PHONY: shell 252 | shell: venv 253 | . $(VENV)/activate && exec $(notdir $(SHELL)) 254 | 255 | .PHONY: bash zsh 256 | bash zsh: venv 257 | . $(VENV)/activate && exec $@ 258 | 259 | 260 | # 261 | # Commandline tools (wildcard rule, executable name must match package name) 262 | # 263 | 264 | ifneq ($(EXE),) 265 | $(VENV)/%: $(VENV)/%$(EXE) ; 266 | .PHONY: $(VENV)/% 267 | .PRECIOUS: $(VENV)/%$(EXE) 268 | endif 269 | 270 | $(VENV)/%$(EXE): $(VENV)/$(MARKER) 271 | $(VENV)/pip install --upgrade $* 272 | $(call touch,$@) 273 | -------------------------------------------------------------------------------- /thor/http/client/exchange.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | from typing import Optional, List, Tuple, TYPE_CHECKING 3 | 4 | from thor.events import EventEmitter 5 | from thor.http.uri import parse_uri 6 | from thor.http.common import ( 7 | States, 8 | Delimiters, 9 | idempotent_methods, 10 | header_names, 11 | RawHeaderListType, 12 | OriginType, 13 | ) 14 | from thor.http.error import ( 15 | UrlError, 16 | ConnectError, 17 | AccessError, 18 | HttpError, 19 | DnsError, 20 | ) 21 | 22 | if TYPE_CHECKING: 23 | from thor.loop import ScheduledEvent 24 | from thor.tcp import TcpConnection 25 | from .client import HttpClient 26 | from .connection import HttpClientConnection 27 | 28 | req_rm_hdrs = [ 29 | b"connection", 30 | b"keep-alive", 31 | b"proxy-authenticate", 32 | b"proxy-authorization", 33 | b"te", 34 | b"trailers", 35 | b"transfer-encoding", 36 | b"upgrade", 37 | b"host", 38 | ] 39 | 40 | 41 | class HttpClientExchange(EventEmitter): 42 | def __init__(self, client: HttpClient) -> None: 43 | EventEmitter.__init__(self) 44 | self.client = client 45 | self.careful = client.careful 46 | self.method: Optional[bytes] = None 47 | self.uri: Optional[bytes] = None 48 | self.req_hdrs: RawHeaderListType = [] 49 | self.req_target: Optional[bytes] = None 50 | self.authority: Optional[bytes] = None 51 | self.res_version: Optional[bytes] = None 52 | self.conn: Optional[HttpClientConnection] = None 53 | self.origin: Optional[OriginType] = None 54 | self._req_body = False 55 | self._req_started = False 56 | self._error_sent = False 57 | self._retries = 0 58 | self._output_q: List[Tuple] = [] 59 | self._response_complete = False 60 | 61 | def __repr__(self) -> str: 62 | status = [self.__class__.__module__ + "." + self.__class__.__name__] 63 | method = (self.method or b"-").decode("utf-8", "replace") 64 | uri = (self.uri or b"-").decode("utf-8", "replace") 65 | status.append(f"{method} <{uri}>") 66 | if self.conn: 67 | status.append(self.conn.tcp_connected and "connected" or "disconnected") 68 | return f"<{', '.join(status)} at {id(self):#x}>" 69 | 70 | def request_start( 71 | self, method: bytes, uri: bytes, req_hdrs: RawHeaderListType 72 | ) -> None: 73 | """ 74 | Start a request to uri using method, where req_hdrs is a list of (field_name, field_value) 75 | for the request headers. 76 | """ 77 | self.method = method 78 | self.uri = uri 79 | self.req_hdrs = req_hdrs 80 | try: 81 | (scheme, host, port, authority, req_target) = parse_uri(self.uri) 82 | self.origin = (scheme, host, port) 83 | self.authority = authority 84 | self.req_target = req_target 85 | except UrlError as why: 86 | self.input_error(why, False) 87 | return 88 | except (TypeError, ValueError): 89 | self.input_error(UrlError("Invalid URL"), False) 90 | return 91 | self.client.attach_conn( 92 | self.origin, self.handle_connect, self.handle_connect_error 93 | ) 94 | 95 | def _req_start(self) -> None: 96 | """ 97 | Queue the request headers for sending. 98 | """ 99 | if self._req_started or self._error_sent: 100 | return 101 | self._req_started = True 102 | req_hdrs = [i for i in self.req_hdrs if not i[0].lower() in req_rm_hdrs] 103 | assert self.authority, "authority not found in _req_start" 104 | req_hdrs.append((b"Host", self.authority)) 105 | if self.client.idle_timeout == 0: 106 | req_hdrs.append((b"Connection", b"close")) 107 | if b"content-length" in header_names(req_hdrs): 108 | delimit = Delimiters.COUNTED 109 | elif self._req_body: 110 | req_hdrs.append((b"Transfer-Encoding", b"chunked")) 111 | delimit = Delimiters.CHUNKED 112 | else: 113 | delimit = Delimiters.NOBODY 114 | 115 | top_line = b"%s %s HTTP/1.1" % (self.method, self.req_target) 116 | if self.conn: 117 | self.conn.output_start(top_line, req_hdrs, delimit) 118 | else: 119 | self._output_q.append(("start", top_line, req_hdrs, delimit)) 120 | 121 | def request_body(self, chunk: bytes) -> None: 122 | "Send part of the request body. May be called zero to many times." 123 | self._req_body = True 124 | self._req_start() 125 | if self.conn: 126 | self.conn.output_body(chunk) 127 | else: 128 | self._output_q.append(("body", chunk)) 129 | 130 | def request_done(self, trailers: RawHeaderListType) -> None: 131 | """ 132 | Signal the end of the request, whether or not there was a body. MUST 133 | be called exactly once for each request. 134 | """ 135 | self._req_start() 136 | if self.conn: 137 | close = self.conn.output_end(trailers) 138 | if close: 139 | self.client.dead_conn(self.conn) 140 | else: 141 | self._output_q.append(("end", trailers)) 142 | 143 | def res_body_pause(self, paused: bool) -> None: 144 | "Temporarily stop / restart sending the response body." 145 | if self.conn and self.conn.tcp_connected: 146 | self.conn.tcp_conn.pause(paused) 147 | 148 | def handle_connect(self, conn: HttpClientConnection) -> None: 149 | "The connection has succeeded." 150 | self.conn = conn 151 | self.conn.attach(self) 152 | for item in self._output_q: 153 | if item[0] == "start": 154 | self.conn.output_start(*item[1:]) 155 | elif item[0] == "body": 156 | self.conn.output_body(*item[1:]) 157 | elif item[0] == "end": 158 | close = self.conn.output_end(*item[1:]) 159 | if close: 160 | self.client.dead_conn(self.conn) 161 | self._output_q = [] 162 | 163 | def handle_connect_error(self, err_type: str, err_id: int, err_str: str) -> None: 164 | "The connection has failed." 165 | 166 | if err_type == "gai": 167 | self.input_error(DnsError(err_str), False) 168 | elif err_type == "access": 169 | self.input_error(AccessError(err_str), False) 170 | elif err_type == "retry": 171 | self.input_error(ConnectError(err_str), False) 172 | elif self._retries < self.client.retry_limit: 173 | self.client.loop.schedule(self.client.retry_delay, self._retry) 174 | else: 175 | self.input_error(ConnectError(err_str), False) 176 | 177 | def conn_closed(self, state: States, delimit: Delimiters) -> None: 178 | "The server closed the connection." 179 | 180 | if self._response_complete: 181 | return 182 | 183 | if state in [States.QUIET, States.ERROR]: 184 | pass 185 | elif delimit == Delimiters.CLOSE: 186 | if self.conn: 187 | self.conn.input_end([]) 188 | elif state == States.WAITING: 189 | if self.method in idempotent_methods: 190 | if self._retries < self.client.retry_limit: 191 | self.client.loop.schedule(self.client.retry_delay, self._retry) 192 | else: 193 | self.input_error( 194 | ConnectError(f"Tried to connect {self._retries + 1} times."), 195 | False, 196 | ) 197 | else: 198 | assert self.method, "method not found in conn_closed" 199 | self.input_error( 200 | ConnectError( 201 | f"Can't retry {self.method.decode('utf-8', 'replace')} method" 202 | ), 203 | False, 204 | ) 205 | else: 206 | self.input_error( 207 | ConnectError( 208 | "Server dropped connection before the response was complete." 209 | ), 210 | False, 211 | ) 212 | 213 | def _retry(self) -> None: 214 | "Retry the request." 215 | self._retries += 1 216 | assert self.origin, "origin not found in _retry" 217 | self.client.attach_conn( 218 | self.origin, self.handle_connect, self.handle_connect_error 219 | ) 220 | 221 | def req_body_pause(self, paused: bool) -> None: 222 | "The client needs the application to pause/unpause the request body." 223 | self.emit("pause", paused) 224 | 225 | # Methods called by HttpClientConnection 226 | 227 | def input_end_notify(self, trailers: RawHeaderListType) -> None: 228 | "Indicate that the response body is complete." 229 | self._response_complete = True 230 | self.emit("response_done", trailers) 231 | 232 | def input_error(self, err: HttpError, close: bool = True) -> None: 233 | "Indicate an error state." 234 | self._error_sent = True 235 | self.emit("error", err) 236 | -------------------------------------------------------------------------------- /doc/http.md: -------------------------------------------------------------------------------- 1 | # HTTP 2 | 3 | Thor provides both a HTTP/1.1 client and server implementation. They support most HTTP features, such as: 4 | 5 | * Persistent Connections (HTTP/1.0-style as well as 1.1) 6 | * Headers across multiple lines 7 | * Chunked requests 8 | 9 | 10 | ## thor.http.HttpClient ( _[thor.loop](loop.md)_ `loop`?, _int_ `idle_timeout`? ) 11 | 12 | Instantiates a HTTP client. If _loop_ is supplied, it will be used as the *thor.loop*; otherwise, the "default" loop will be used. 13 | 14 | HTTP clients share a pool of idle connections. 15 | 16 | There are several settings available as class variables: 17 | 18 | * _thor.TcpClient_ `HttpClient.tcp_client_class` - what to use as a TCP client. 19 | * _int_ or _None_ `HttpClient.connect_timeout` - connect timeout, in seconds. Default `None`. 20 | * _int_ or _None_ `HttpClient.read_timeout` - timeout between reads on an active connection, in seconds. Default `None`. 21 | * _int_ or _None_ `HttpClient.idle_timeout` - how long idle persistent connections are left open, in seconds. Default `60`; `None` to disable. 22 | * _int_ `HttpClient.retry_limit` - How many additional times to try a request that fails (e.g., dropped connection). Default `2`. 23 | * _int_ `HttpClient.retry_delay` - how long to wait between retries, in seconds (or fractions thereof). Default `0.5`. 24 | 25 | 26 | ### _thor.http.HttpClientExchange_ thor.http.HttpClient.exchange () 27 | 28 | Create a request/response exchange. 29 | 30 | ### thor.http.HttpClientExchange 31 | 32 | #### _void_ request\_start ( _bytes_ `method`, _bytes_ `uri`, _[headers](#headers)_ `headers` ) 33 | 34 | Start the request to `uri` using `method` and a list of tuples `headers` (see [working with HTTP headers](#headers)). 35 | 36 | note that `uri` is a string; if you want to use an IRI, you'll need to convert it first. 37 | 38 | Also, hop-by-hop headers will be stripped from `headers`; Thor manages its own connections headers (such as _Connection_, _Keep-Alive_, and so on.) 39 | 40 | After calling *request_start*, *request_body* may be called zero or more times, and then *request_done* must be called. 41 | 42 | 43 | #### _void_ request\_body ( _bytes_ `chunk` ) 44 | 45 | Send a `chunk` of request body content. 46 | 47 | 48 | #### _void_ request\_done ( _[headers](#headers)_ `trailers`? ) 49 | 50 | Signal that the request body is finished. This must be called for every request. `trailers` is the list of HTTP trailers; see [working with HTTP headers](#headers). 51 | 52 | 53 | #### event 'response\_start' ( _bytes_ `status`, _bytes_ `phrase`, _[headers](#headers)_ `headers` ) 54 | 55 | Emitted once, when the client starts receiving the exchange's response. `status` and `phrase` contain the HTTP response status code and reason phrase, respectively, and `headers` contains the response header tuples (see [working with HTTP headers](#headers)). 56 | 57 | 58 | #### event 'response\_nonfinal' ( _bytes_ `status`, _bytes_ `phrase`, _[headers](#headers)_ `headers` ) 59 | 60 | Emitted zero to many times, when the client receives a non-final (1xx) HTTP response. `status`, `phrase` and `headers` are as described for *response_start*. 61 | 62 | 63 | #### event 'response\_body' ( _bytes_ `chunk` ) 64 | 65 | Emitted zero to many times, when a `chunk` of the response body is received. 66 | 67 | 68 | #### event 'response\_done' ( _[headers](#headers)_ `trailers` ) 69 | 70 | Emitted once, when the response is successfully completed. `trailers` is the list 71 | of HTTP trailers; see [working with HTTP headers](#headers). 72 | 73 | 74 | #### event 'error' ( _[thor.http.error.HttpError](error.md)_ `err` ) 75 | 76 | Emitted when there is an error with the request or response. `err` is an instance of one of the *thor.http.error* classes that describes what happened. 77 | 78 | If _err.client_recoverable_ is `False`, no other events will be emitted by this exchange. 79 | 80 | 81 | 82 | ## thor.http.HttpServer ( _bytes_ `host`, _int_ `port`, _[thor.loop](loop.md)_ `loop`? ) 83 | 84 | Creates a new server listening on `host`:`port`. If `loop` is supplied, it will be used as the *thor.loop*; otherwise, the "default" loop will be used. 85 | 86 | The following settings are available as class variables: 87 | 88 | * _thor.TcpServer_ `HttpServer.tcp_server_class` - what to use as a TCP server. 89 | * _int_ or _None_ `HttpServer.idle_timeout` - how long idle persistent connections are left open, in seconds. Default 60; None to disable. 90 | 91 | ### event 'start' () 92 | 93 | Emitted when the server starts. 94 | 95 | ### event 'stop' () 96 | 97 | Emitted when the server stops. 98 | 99 | 100 | ### event 'exchange' ( _thor.http.HttpServerExchange_ `exchange` ) 101 | 102 | Emitted when the server starts a new request/response `exchange`. 103 | 104 | 105 | ### _void_ thor.http.HttpServer.shutdown () 106 | 107 | Stops the server from accepting new connections. Existing connections will remain active. 108 | 109 | 110 | ### _void_ thor.http.HttpServer.graceful_shutdown () 111 | 112 | Stops the server from accepting new connections, and waits for all active connections to close before emitting [stop](#event-stop-). 113 | 114 | 115 | ### thor.http.HttpServerExchange 116 | 117 | 118 | #### event 'request\_start' ( _bytes_ `method`, _bytes_ `uri`, _[headers](#headers)_ `headers` ) 119 | 120 | Emitted once, when the exchange receives a request to `uri` using `method` and a list of tuples `headers` (see [working with HTTP headers](#headers)). 121 | 122 | 123 | #### event 'request\_body' ( _bytes_ `chunk` ) 124 | 125 | Emitted zero to many times, when a `chunk` of the request body is received. 126 | 127 | 128 | #### event 'request\_done' ( _[headers](#headers)_ `trailers`? ) 129 | 130 | Emitted once, when the request is successfully completed. `trailers` is the list of HTTP trailers; see [working with HTTP headers](#headers). 131 | 132 | 133 | #### _void_ response\_start ( _bytes_ `status`, _bytes_ `phrase`, _[headers](#headers)_ `headers` ) 134 | 135 | Start sending the exchange's response. `status` and `phrase` should contain the HTTP response status code and reason phrase, respectively, and `headers` should contain the response header tuples (see [working with HTTP headers](#headers)). 136 | 137 | Note that hop-by-hop headers will be stripped from `headers`; Thor manages its own connections headers (such as _Connection_, _Keep-Alive_, and so on.) 138 | 139 | 140 | #### _void_ response\_nonfinal ( _bytes_ `status`, _bytes_ `phrase`, _[headers](#headers)_ `headers` ) 141 | 142 | Send a non-final (1xx) HTTP response. This can be called zero to many times before calling *response_start*. 143 | 144 | 145 | #### _void_ response\_body ( _bytes_ `chunk` ) 146 | 147 | Send a `chunk` of response body content. 148 | 149 | 150 | #### _void_ response\_done ( _[headers](#headers)_ `trailers` ) 151 | 152 | Signal that the response body is finished. This must be called for every response. `trailers` is the list of HTTP trailers; see [working with HTTP headers](#headers). 153 | 154 | 155 | 156 | 157 | 158 | ## Working with HTTP Headers 159 | 160 | In Thor's HTTP APIs, headers are moved around as lists of tuples, where each tuple is a (_bytes_ `field-name`, _bytes_ `field-value`) pair. For example: 161 | 162 | [ 163 | ("Content-Type", b" text/plain"), 164 | ("Foo", b"bar, baz"), 165 | ("Cache-Control", b" max-age=30, must-revalidate"), 166 | ("Foo", b"boom"), 167 | ("user-agent", b"Foo/1.0") 168 | ] 169 | 170 | This is an intentionally low-level representation of HTTP headers; each tuple corresponds to one on-the-wire line, in order. That means that a field-name can appear more than once (note that 'Foo' appears twice above), and that multiple values can appear in one field-value (note the "Foo" and "Cache-Control" headers above). Whitespace can appear at the beginning of field-values, and field-names are not case-normalised. 171 | 172 | Thor has several utility functions for manipulating this data structure; see [thor.http.header_names](#header_names), [thor.http.header_dict](#header_dict), and [thor.http.get_header](#get_header) 173 | 174 | 175 | 176 | 177 | ### _set_ thor.http.header\_names ( _[headers](#headers)_ `headers` ) 178 | 179 | Given a list of header tuples `headers`, return the set of _bytes_ header field-names present. 180 | 181 | 182 | 183 | 184 | ### _dict_ thor.http.header\_dict ( _[headers](#headers)_ `headers`, _list_ `omit` ) 185 | 186 | Given a list of header tuples `headers`, return a dictionary whose keys are the _bytes_ header field-names (normalised to lower case) and whose values are lists of _bytes_ field-values. 187 | 188 | Note that header field-values containing commas are split into separate values. Therefore, this function is NOT suitable for use on fields whose values may contain commas (e.g., in quoted strings, or in cookie values). 189 | 190 | If `omit`, a list of _bytes_ field-names, is specified, those field names will be omitted from the dictionary. 191 | 192 | 193 | 194 | 195 | ### _list_ thor.http.get\_header ( _[headers](#headers)_ `headers`, _bytes_ `fieldname` ) 196 | 197 | Given a list of header tuples `headers`, return a list of _bytes_ field-values for the given `fieldname`. 198 | 199 | Note that header field-values containing commas are split into separate values. Therefore, this function is NOT suitable for use on fields whose values may contain commas (e.g., in quoted strings, or in cookie values). 200 | 201 | 202 | -------------------------------------------------------------------------------- /test/test_http_server.py: -------------------------------------------------------------------------------- 1 | ##!/usr/bin/env python 2 | 3 | import socket 4 | import sys 5 | import time 6 | import unittest 7 | 8 | import framework 9 | 10 | import thor 11 | from thor.events import on 12 | from thor.http import HttpServer 13 | 14 | 15 | class TestHttpServer(framework.ClientServerTestCase): 16 | def create_server(self, server_side): 17 | server = HttpServer(framework.test_host, 0, loop=self.loop) 18 | test_port = server.tcp_server.sock.getsockname()[1] 19 | server_side(server) 20 | 21 | def stop(): 22 | server.shutdown() 23 | 24 | return (stop, test_port) 25 | 26 | def create_client(self, test_host, test_port, client_side): 27 | def run_client(client_side1): 28 | client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 29 | client.connect((test_host, test_port)) 30 | client_side1(client, test_host, test_port) 31 | client.close() 32 | 33 | self.move_to_thread(target=run_client, args=(client_side,)) 34 | 35 | def check_exchange(self, exchange, expected): 36 | """ 37 | Given an exchange, check that the status, phrase and body are as 38 | expected, and verify that it actually happened. 39 | """ 40 | exchange.test_happened = False 41 | 42 | @on(exchange) 43 | def error(err_msg): 44 | exchange.test_happened = True 45 | self.assertEqual(err_msg, expected.get("error", err_msg)) 46 | self.loop.stop() 47 | 48 | @on(exchange) 49 | def request_start(method, uri, headers): 50 | self.assertEqual(method, expected.get("method", method)) 51 | self.assertEqual(uri, expected.get("phrase", uri)) 52 | 53 | exchange.tmp_req_body = b"" 54 | 55 | @on(exchange) 56 | def request_body(chunk): 57 | exchange.tmp_req_body += chunk 58 | 59 | @on(exchange) 60 | def request_done(trailers): 61 | exchange.test_happened = True 62 | self.assertEqual(trailers, expected.get("req_trailers", trailers)) 63 | self.assertEqual( 64 | exchange.tmp_req_body, expected.get("body", exchange.tmp_req_body) 65 | ) 66 | self.loop.stop() 67 | 68 | @on(self.loop) 69 | def stop(): 70 | self.assertTrue(exchange.test_happened) 71 | 72 | def test_basic(self): 73 | def server_side(server): 74 | def check(exchange): 75 | self.check_exchange(exchange, {"method": b"GET", "uri": b"/"}) 76 | 77 | server.on("exchange", check) 78 | 79 | def client_side(client_conn, test_host, test_port): 80 | client_conn.sendall( 81 | b"""\ 82 | GET / HTTP/1.1 83 | Host: %s:%i 84 | 85 | """ 86 | % (test_host, test_port) 87 | ) 88 | time.sleep(0.1) 89 | client_conn.close() 90 | 91 | self.go([server_side], [client_side]) 92 | 93 | def test_extraline(self): 94 | def server_side(server): 95 | def check(exchange): 96 | self.check_exchange(exchange, {"method": b"GET", "uri": b"/"}) 97 | 98 | server.on("exchange", check) 99 | 100 | def client_side(client_conn, test_host, test_port): 101 | client_conn.sendall( 102 | b"""\ 103 | 104 | GET / HTTP/1.1 105 | Host: %s:%i\r 106 | \r 107 | """ 108 | % (test_host, test_port) 109 | ) 110 | time.sleep(0.1) 111 | client_conn.close() 112 | 113 | self.go([server_side], [client_side]) 114 | 115 | def test_post(self): 116 | def server_side(server): 117 | def check(exchange): 118 | self.check_exchange(exchange, {"method": b"POST", "uri": b"/foo"}) 119 | 120 | server.on("exchange", check) 121 | 122 | def client_side(client_conn, test_host, test_port): 123 | client_conn.sendall( 124 | b"""\ 125 | POST / HTTP/1.1 126 | Host: %s:%i 127 | Content-Type: text/plain 128 | Content-Length: 5 129 | 130 | 12345""" 131 | % (test_host, test_port) 132 | ) 133 | time.sleep(0.1) 134 | client_conn.close() 135 | 136 | self.go([server_side], [client_side]) 137 | 138 | def test_post_extra_crlf(self): 139 | def server_side(server): 140 | def check(exchange): 141 | self.check_exchange(exchange, {"method": b"POST", "uri": b"/foo"}) 142 | 143 | server.on("exchange", check) 144 | 145 | def client_side(client_conn, test_host, test_port): 146 | client_conn.sendall( 147 | b"""\ 148 | POST / HTTP/1.1 149 | Host: %s:%i 150 | Content-Type: text/plain 151 | Content-Length: 5 152 | 153 | 12345 154 | """ 155 | % (test_host, test_port) 156 | ) 157 | time.sleep(0.1) 158 | client_conn.close() 159 | 160 | self.go([server_side], [client_side]) 161 | 162 | def test_1xx_response(self): 163 | def server_side(server): 164 | def check(exchange): 165 | @on(exchange, "request_done") 166 | def on_request_done(trailers): 167 | exchange.response_nonfinal( 168 | b"103", 169 | b"Early Hints", 170 | [(b"Link", b"; rel=preload; as=style")], 171 | ) 172 | exchange.response_start(b"200", b"OK", [(b"Content-Length", b"5")]) 173 | exchange.response_body(b"hello") 174 | exchange.response_done([]) 175 | self.exchange_handled = True 176 | 177 | server.on("exchange", check) 178 | 179 | def client_side(client_conn, test_host, test_port): 180 | client_conn.sendall( 181 | b"GET / HTTP/1.1\r\nHost: %s:%i\r\n\r\n" % (test_host, test_port) 182 | ) 183 | res = b"" 184 | client_conn.settimeout(2) 185 | try: 186 | while True: 187 | chunk = client_conn.recv(1024) 188 | if not chunk: 189 | break 190 | res += chunk 191 | if b"hello" in res: 192 | break 193 | except socket.timeout: 194 | pass 195 | self.res = res 196 | self.loop.stop() 197 | 198 | self.exchange_handled = False 199 | self.res = b"" 200 | self.go([server_side], [client_side]) 201 | self.assertTrue(self.exchange_handled) 202 | self.assertIn(b"HTTP/1.1 103 Early Hints", self.res) 203 | self.assertIn(b"HTTP/1.1 200 OK", self.res) 204 | self.assertIn(b"hello", self.res) 205 | 206 | def test_reentrancy(self): 207 | def server_side(server): 208 | def check(exchange): 209 | @on(exchange, "request_done") 210 | def on_request_done(trailers): 211 | ex = exchange 212 | ex.response_start(b"200", b"OK", []) 213 | ex.response_body(b"done") 214 | ex.response_done([]) 215 | self.exchange_handled = True 216 | self.loop.stop() 217 | 218 | server.on("exchange", check) 219 | 220 | def client_side(client_conn, test_host, test_port): 221 | client_conn.sendall( 222 | b"""\ 223 | POST / HTTP/1.1 224 | Host: %s:%i 225 | Content-Length: 0 226 | 227 | """ 228 | % (test_host, test_port) 229 | ) 230 | time.sleep(0.1) 231 | client_conn.close() 232 | 233 | self.exchange_handled = False 234 | self.go([server_side], [client_side]) 235 | self.assertTrue(self.exchange_handled) 236 | 237 | 238 | # def test_pipeline(self): 239 | # def server_side(server): 240 | # server.ex_count = 0 241 | # def check(exchange): 242 | # self.check_exchange(exchange, { 243 | # 'method': 'GET', 244 | # 'uri': '/' 245 | # }) 246 | # server.ex_count += 1 247 | # server.on('exchange', check) 248 | # @on(self.loop) 249 | # def stop(): 250 | # self.assertEqual(server.ex_count, 2) 251 | # 252 | # def client_side(client_conn, test_host, test_port): 253 | # client_conn.sendall("""\ 254 | # GET / HTTP/1.1 255 | # Host: %s:%i 256 | # 257 | # GET / HTTP/1.1 258 | # Host: %s:%i 259 | # 260 | # """ % ( 261 | # test_host, test_port, 262 | # test_host, test_port 263 | # )) 264 | # time.sleep(0.1) 265 | # client_conn.close() 266 | # self.go([server_side], [client_side]) 267 | 268 | 269 | # def test_conn_close(self): 270 | # def test_req_nobody(self): 271 | # def test_res_nobody(self): 272 | # def test_bad_http_version(self): 273 | # def test_pause(self): 274 | # def test_extra_crlf_after_post(self): 275 | # def test_absolute_uri(self): # ignore host header 276 | # def test_host_header(self):# 277 | # def test_unknown_transfercode(self): # should be 501 278 | # def test_shutdown(self): 279 | # def test_alternate_tcp_server(self): 280 | # def test_startline_encoding(self): 281 | 282 | 283 | if __name__ == "__main__": 284 | unittest.main() 285 | -------------------------------------------------------------------------------- /thor/http/server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Thor HTTP Server 5 | 6 | This library allow implementation of an HTTP/1.1 server that is 7 | "non-blocking," "asynchronous" and "event-driven" -- i.e., it achieves very 8 | high performance and concurrency, so long as the application code does not 9 | block (e.g., upon network, disk or database access). Blocking on one request 10 | will block the entire server. 11 | 12 | """ 13 | 14 | import os 15 | import sys 16 | from typing import Optional, List, Tuple, Any 17 | 18 | from thor.events import EventEmitter, on 19 | from thor.loop import LoopBase, ScheduledEvent 20 | from thor.tcp import TcpServer, TcpConnection 21 | 22 | from thor.http.common import ( 23 | HttpMessageHandler, 24 | States, 25 | Delimiters, 26 | hop_by_hop_hdrs, 27 | get_header, 28 | header_names, 29 | ) 30 | from thor.http.error import ( 31 | HttpError, 32 | HttpVersionError, 33 | HostRequiredError, 34 | TransferCodeError, 35 | ) 36 | 37 | RawHeaderListType = List[Tuple[bytes, bytes]] 38 | 39 | 40 | class HttpServer(EventEmitter): 41 | "An asynchronous HTTP server." 42 | 43 | tcp_server_class = TcpServer 44 | idle_timeout = 60 # in seconds 45 | 46 | def __init__(self, host: bytes, port: int, loop: Optional[LoopBase] = None) -> None: 47 | EventEmitter.__init__(self) 48 | self.tcp_server = self.tcp_server_class(host, port, loop=loop) 49 | self.loop = self.tcp_server.loop 50 | self.tcp_server.on("connect", self.handle_conn) 51 | self.loop.schedule(0, self.emit, "start") 52 | self.connections: List["HttpServerConnection"] = [] 53 | self.shutting_down = False 54 | 55 | def handle_conn(self, tcp_conn: TcpConnection) -> None: 56 | http_conn = HttpServerConnection(tcp_conn, self) 57 | self.connections.append(http_conn) 58 | tcp_conn.on("data", http_conn.handle_input) 59 | tcp_conn.on("disconnect", http_conn.conn_closed) 60 | tcp_conn.on("pause", http_conn.res_body_pause) 61 | tcp_conn.pause(False) 62 | 63 | def graceful_shutdown(self) -> None: 64 | """ 65 | Stop the server gracefully. 66 | 67 | This stops accepting new connections and waits for all active 68 | connections to close before emitting "stop". 69 | """ 70 | self.shutting_down = True 71 | self.tcp_server.on("stop", lambda: self.emit("stop")) 72 | self.tcp_server.graceful_shutdown() 73 | for conn in list(self.connections): 74 | if conn.is_idle: 75 | conn.close_conn() 76 | 77 | def shutdown(self) -> None: 78 | "Stop the server" 79 | self.tcp_server.shutdown() 80 | self.emit("stop") 81 | 82 | 83 | class HttpServerConnection(HttpMessageHandler, EventEmitter): 84 | "A handler for an HTTP server connection." 85 | 86 | default_state = States.WAITING 87 | 88 | def __init__(self, tcp_conn: TcpConnection, server: HttpServer) -> None: 89 | HttpMessageHandler.__init__(self) 90 | EventEmitter.__init__(self) 91 | self.tcp_conn: Optional[TcpConnection] = tcp_conn 92 | self.server = server 93 | self.ex_queue: List[HttpServerExchange] = [] # queue of exchanges 94 | self.output_paused = False 95 | self.output_paused = False 96 | self._idler: Optional[ScheduledEvent] = self.server.loop.schedule( 97 | self.server.idle_timeout, self.close_conn 98 | ) 99 | 100 | @property 101 | def is_idle(self) -> bool: 102 | return len(self.ex_queue) == 0 103 | 104 | def req_body_pause(self, paused: bool) -> None: 105 | """ 106 | Indicate that the server should pause (True) or unpause (False) the 107 | request. 108 | """ 109 | if self.tcp_conn: 110 | self.tcp_conn.pause(paused) 111 | 112 | def close_conn(self) -> None: 113 | "Close the connection." 114 | if self.tcp_conn and self.tcp_conn.tcp_connected: 115 | self.tcp_conn.close() 116 | self.tcp_conn = None 117 | 118 | def exchange_done(self, exchange: "HttpServerExchange") -> None: 119 | exchange.res_complete = True 120 | if exchange.req_complete: 121 | if exchange in self.ex_queue: 122 | self.ex_queue.remove(exchange) 123 | if self.is_idle and self.server.shutting_down: 124 | self.close_conn() 125 | 126 | # Methods called by tcp 127 | 128 | def res_body_pause(self, paused: bool) -> None: 129 | "Pause/unpause sending the response body." 130 | self.output_paused = paused 131 | self.emit("pause", paused) 132 | if not paused: 133 | self.drain_exchange_queue() 134 | 135 | def conn_closed(self) -> None: 136 | "The server connection has closed." 137 | if self in self.server.connections: 138 | self.server.connections.remove(self) 139 | self.ex_queue = [] 140 | self.tcp_conn = None 141 | 142 | # Methods called by common.HttpRequestHandler 143 | 144 | def output(self, data: bytes) -> None: 145 | if self.tcp_conn and self.tcp_conn.tcp_connected: 146 | self.tcp_conn.write(data) 147 | 148 | def output_done(self) -> None: 149 | self._idler = self.server.loop.schedule( 150 | self.server.idle_timeout, self.close_conn 151 | ) 152 | 153 | def input_start( 154 | self, 155 | top_line: bytes, 156 | hdr_tuples: RawHeaderListType, 157 | conn_tokens: List[bytes], 158 | transfer_codes: List[bytes], 159 | content_length: Optional[int], 160 | ) -> Tuple[bool, bool]: 161 | """ 162 | Take the top set of headers from the input stream, parse them 163 | and queue the request to be processed by the application. 164 | """ 165 | if self._idler: 166 | self._idler.delete() 167 | self._idler = None 168 | try: 169 | method, req_line = top_line.split(None, 1) 170 | uri, req_version = req_line.rsplit(None, 1) 171 | req_version = req_version.rsplit(b"/", 1)[1] 172 | except (ValueError, IndexError): 173 | self.input_error(HttpVersionError(top_line.decode("utf-8", "replace"))) 174 | raise ValueError 175 | if b"host" not in header_names(hdr_tuples): 176 | self.input_error(HostRequiredError()) 177 | raise ValueError 178 | for code in transfer_codes: 179 | # we only support 'identity' and chunked' codes in requests 180 | if code not in [b"identity", b"chunked"]: 181 | self.input_error(TransferCodeError(code.decode("utf-8", "replace"))) 182 | raise ValueError 183 | exchange = HttpServerExchange(self, method, uri, hdr_tuples, req_version) 184 | self.ex_queue.append(exchange) 185 | self.server.emit("exchange", exchange) 186 | if not self.output_paused: 187 | # we only start new requests if we have some output buffer 188 | # available. 189 | exchange.request_start() 190 | allows_body = bool(content_length and content_length > 0) or ( 191 | transfer_codes != [] 192 | ) 193 | return allows_body, True 194 | 195 | def input_body(self, chunk: bytes) -> None: 196 | "Process a request body chunk from the wire." 197 | self.ex_queue[-1].emit("request_body", chunk) 198 | 199 | def input_end(self, trailers: RawHeaderListType) -> None: 200 | "Indicate that the request body is complete." 201 | ex = self.ex_queue[-1] 202 | ex.req_complete = True 203 | ex.emit("request_done", trailers) 204 | if ex.res_complete and ex in self.ex_queue: 205 | self.ex_queue.remove(ex) 206 | 207 | def input_error(self, err: HttpError) -> None: 208 | """ 209 | Indicate a parsing problem with the request body (which 210 | hasn't been queued as an exchange yet). 211 | """ 212 | if err.server_recoverable: 213 | self.emit("error", err) 214 | else: 215 | self._input_state = States.ERROR 216 | status_code, status_phrase = err.server_status 217 | hdrs = [(b"Content-Type", b"text/plain")] 218 | body = err.desc.encode("utf-8") 219 | if err.detail: 220 | body += b" (%s)" % err.detail.encode("utf-8") 221 | ex = HttpServerExchange(self, b"", b"", [], b"1.1") 222 | ex.response_start(status_code, status_phrase, hdrs) 223 | ex.response_body(body) 224 | ex.response_done([]) 225 | self.ex_queue.append(ex) 226 | self.close_conn() 227 | 228 | def drain_exchange_queue(self) -> None: 229 | """ 230 | Walk through the exchange queue and kick off unstarted requests 231 | until we run out of output buffer. 232 | """ 233 | for exchange in self.ex_queue: 234 | if not exchange.started: 235 | exchange.request_start() 236 | 237 | 238 | class HttpServerExchange(EventEmitter): 239 | """ 240 | A request/response interaction on an HTTP server. 241 | """ 242 | 243 | def __init__( 244 | self, 245 | http_conn: HttpServerConnection, 246 | method: bytes, 247 | uri: bytes, 248 | req_hdrs: RawHeaderListType, 249 | req_version: bytes, 250 | ) -> None: 251 | EventEmitter.__init__(self) 252 | self.http_conn = http_conn 253 | self.method = method 254 | self.uri = uri 255 | self.req_hdrs = req_hdrs 256 | self.req_version = req_version 257 | self.started = False 258 | self.req_complete = False 259 | self.res_complete = False 260 | 261 | def __repr__(self) -> str: 262 | status = [self.__class__.__module__ + "." + self.__class__.__name__] 263 | method = self.method.decode("ascii") or "-" 264 | uri = self.uri.decode("utf-8", "replace") or "-" 265 | status.append(f"{method} <{uri}>") 266 | return f'<{", ".join(status)} at {id(self):#x}>' 267 | 268 | def request_start(self) -> None: 269 | self.started = True 270 | self.emit("request_start", self.method, self.uri, self.req_hdrs) 271 | 272 | def response_nonfinal( 273 | self, status_code: bytes, status_phrase: bytes, res_hdrs: RawHeaderListType 274 | ) -> None: 275 | "Send a non-final (1xx) response. Can be called zero to many times." 276 | self.http_conn.output_start( 277 | b"HTTP/1.1 %s %s" % (status_code, status_phrase), 278 | res_hdrs, 279 | Delimiters.NONE, 280 | is_final=False, 281 | ) 282 | 283 | def response_start( 284 | self, status_code: bytes, status_phrase: bytes, res_hdrs: RawHeaderListType 285 | ) -> None: 286 | "Start a response. Must only be called once per response." 287 | res_hdrs = [i for i in res_hdrs if not i[0].lower() in hop_by_hop_hdrs] 288 | try: 289 | body_len = int(get_header(res_hdrs, b"content-length").pop(0)) 290 | except (IndexError, ValueError): 291 | body_len = None 292 | if body_len is not None: 293 | delimit = Delimiters.COUNTED 294 | res_hdrs.append((b"Connection", b"keep-alive")) 295 | elif self.req_version == b"1.1": 296 | delimit = Delimiters.CHUNKED 297 | res_hdrs.append((b"Transfer-Encoding", b"chunked")) 298 | else: 299 | delimit = Delimiters.CLOSE 300 | res_hdrs.append((b"Connection", b"close")) 301 | 302 | self.http_conn.output_start( 303 | b"HTTP/1.1 %s %s" % (status_code, status_phrase), 304 | res_hdrs, 305 | delimit, 306 | is_final=True, 307 | ) 308 | 309 | def response_body(self, chunk: bytes) -> None: 310 | "Send part of the response body. May be called zero to many times." 311 | self.http_conn.output_body(chunk) 312 | 313 | def response_done(self, trailers: RawHeaderListType) -> None: 314 | """ 315 | Signal the end of the response, whether or not there was a body. MUST 316 | be called exactly once for each response. 317 | """ 318 | close = self.http_conn.output_end(trailers) 319 | self.http_conn.exchange_done(self) 320 | if close and self.http_conn.tcp_conn: 321 | self.http_conn.close_conn() 322 | 323 | 324 | def test_handler(ex: HttpServerExchange) -> None: # pragma: no cover 325 | @on(ex, "request_start") 326 | def go(*args: Any) -> None: 327 | print(f"start: {str(args[1])} on {id(ex.http_conn)}") 328 | ex.response_start(b"200", b"OK", []) 329 | ex.response_body(b"foo!") 330 | ex.response_done([]) 331 | 332 | @on(ex, "request_body") 333 | def body(chunk: bytes) -> None: 334 | print(f"body: {chunk.decode('utf-8', 'replace')}") 335 | 336 | @on(ex, "request_done") 337 | def done(trailers: RawHeaderListType) -> None: 338 | print(f"done: {str(trailers)}") 339 | 340 | 341 | if __name__ == "__main__": 342 | from thor.loop import run 343 | 344 | sys.stderr.write(f"PID: {os.getpid()}\n") 345 | h, p = b"127.0.0.1", int(sys.argv[1]) 346 | demo_server = HttpServer(h, p) 347 | demo_server.on("exchange", test_handler) 348 | run() 349 | -------------------------------------------------------------------------------- /thor/tcp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | push-based asynchronous TCP 5 | 6 | This is a generic library for building event-based / asynchronous 7 | TCP servers and clients. 8 | 9 | It uses a push model; i.e., the network connection pushes data to 10 | you (using a 'data' event), and you push data to the network connection 11 | (using the write method). 12 | """ 13 | 14 | import errno 15 | import os 16 | import sys 17 | import socket 18 | from typing import Optional, List, Callable, Set 19 | 20 | from thor.dns import DnsResult, Address 21 | from thor.loop import EventSource, LoopBase, schedule 22 | from thor.loop import ScheduledEvent 23 | 24 | 25 | class TcpConnection(EventSource): 26 | """ 27 | An asynchronous TCP connection. 28 | 29 | Emits: 30 | - data (chunk): incoming data 31 | - close (): the other party has closed the connection 32 | - pause (bool): whether the connection has been paused 33 | 34 | It will emit the 'data' even every time incoming data is 35 | available; 36 | 37 | > def process(data): 38 | > print "got some data:", data 39 | > tcp_conn.on('data', process) 40 | 41 | When you want to write to the connection, just write to it: 42 | 43 | > tcp_conn.write(data) 44 | 45 | If you want to close the connection from your side, just call close: 46 | 47 | > tcp_conn.close() 48 | 49 | Note that this will flush any data already written. 50 | 51 | If you want to close the connection immediately, discarding any buffered 52 | data, call abort: 53 | 54 | > tcp_conn.abort() 55 | 56 | If you try to write to a closed connection, an OSError will be raised. 57 | 58 | If the other side closes the connection, The 'close' event will be 59 | emitted; 60 | 61 | > def handle_close(): 62 | > print "oops, they don't like us any more..." 63 | > tcp_conn.on('close', handle_close) 64 | 65 | If you write too much data to the connection and the buffers fill up, 66 | pause_cb will be emitted with True to tell you to stop sending data 67 | temporarily; 68 | 69 | > def handle_pause(paused): 70 | > if paused: 71 | > # stop sending data 72 | > else: 73 | > # it's OK to start again 74 | > tcp_conn.on('pause', handle_pause) 75 | 76 | Note that this is advisory; if you ignore it, the data will still be 77 | buffered, but the buffer will grow. 78 | 79 | Likewise, if you want to pause the connection because your buffers 80 | are full, call pause; 81 | 82 | > tcp_conn.pause(True) 83 | 84 | but don't forget to tell it when it's OK to send data again; 85 | 86 | > tcp_conn.pause(False) 87 | 88 | NOTE that connections are paused to start with; if you want to start 89 | getting data from them, you'll need to pause(False). 90 | """ 91 | 92 | write_bufsize = 16 # number of chunks 93 | read_bufsize = 1024 * 16 # bytes 94 | 95 | block_errs = set([errno.EAGAIN, errno.EWOULDBLOCK, errno.ETIMEDOUT]) 96 | close_errs = set( 97 | [ 98 | errno.EBADF, 99 | errno.ECONNRESET, 100 | errno.ESHUTDOWN, 101 | errno.ECONNABORTED, 102 | errno.ECONNREFUSED, 103 | errno.ENOTCONN, 104 | errno.EPIPE, 105 | ] 106 | ) 107 | 108 | def __init__( 109 | self, sock: socket.socket, address: Address, loop: Optional[LoopBase] = None 110 | ) -> None: 111 | EventSource.__init__(self, loop) 112 | self.socket = sock 113 | self.address = address 114 | self.tcp_connected = True # we assume a connected socket 115 | self._input_paused = True # we start with input paused 116 | self._output_paused = False 117 | self._closing = False 118 | self._write_buffer: List[bytes] = [] 119 | 120 | self.register_fd(sock.fileno()) 121 | self.on("fd_readable", self.handle_readable) 122 | self.on("fd_writable", self.handle_writable) 123 | self.once("fd_close", self._handle_close) 124 | self.event_add("fd_close") 125 | 126 | def __repr__(self) -> str: 127 | status = [self.__class__.__module__ + "." + self.__class__.__name__] 128 | status.append(self.tcp_connected and "connected" or "disconnected") 129 | status.append(f"{self.address[0]}:{self.address[1]}") 130 | if self._input_paused: 131 | status.append("input paused") 132 | if self._output_paused: 133 | status.append("output paused") 134 | if self._closing: 135 | status.append("closing") 136 | if self._write_buffer: 137 | status.append(f"{len(self._write_buffer)} write buffered") 138 | return f"<{', '.join(status)} at {id(self):#x}>" 139 | 140 | def handle_readable(self) -> None: 141 | "The connection has data read for reading" 142 | try: 143 | data = self.socket.recv(self.read_bufsize) 144 | except (socket.error, OSError) as why: 145 | if why.args[0] in self.block_errs: 146 | return 147 | if why.args[0] in self.close_errs: 148 | self._handle_close() 149 | return 150 | raise 151 | if data == b"": 152 | self._handle_close() 153 | else: 154 | self.emit("data", data) 155 | 156 | def handle_writable(self) -> None: 157 | "The connection is ready for writing; write any buffered data." 158 | if self._write_buffer: 159 | data = b"".join(self._write_buffer) 160 | try: 161 | sent = self.socket.send(data) 162 | except (socket.error, OSError) as why: 163 | if why.args[0] in self.block_errs: 164 | return 165 | if why.args[0] in self.close_errs: 166 | self._handle_close() 167 | return 168 | raise 169 | if sent < len(data): 170 | self._write_buffer = [data[sent:]] 171 | else: 172 | self._write_buffer = [] 173 | if self._output_paused and len(self._write_buffer) < self.write_bufsize: 174 | self._output_paused = False 175 | self.emit("pause", False) 176 | if self._closing: 177 | self._close() 178 | if not self._write_buffer: 179 | self.event_del("fd_writable") 180 | 181 | def write(self, data: bytes) -> None: 182 | "Write data to the connection." 183 | if not self.tcp_connected: 184 | raise OSError("Connection closed") 185 | self._write_buffer.append(data) 186 | if len(self._write_buffer) > self.write_bufsize: 187 | self._output_paused = True 188 | self.emit("pause", True) 189 | self.event_add("fd_writable") 190 | 191 | def pause(self, paused: bool) -> None: 192 | """ 193 | Temporarily stop/start reading from the connection and pushing 194 | it to the app. 195 | """ 196 | if paused: 197 | self.event_del("fd_readable") 198 | else: 199 | self.event_add("fd_readable") 200 | self._input_paused = paused 201 | 202 | def close(self) -> None: 203 | "Flush buffered data (if any) and close the connection." 204 | self.pause(True) 205 | if self._write_buffer: 206 | self._closing = True 207 | else: 208 | self._close() 209 | 210 | def abort(self) -> None: 211 | "Close the connection immediately, discarding buffered data." 212 | self._close() 213 | 214 | def _handle_close(self) -> None: 215 | "The connection has been closed by the other side." 216 | self._close() 217 | self.emit("close") 218 | 219 | def _close(self) -> None: 220 | self.emit("disconnect") 221 | self.tcp_connected = False 222 | self.remove_listeners("fd_readable", "fd_writable", "fd_close") 223 | self.unregister_fd() 224 | if self.socket: 225 | self.socket.close() 226 | 227 | 228 | class TcpServer(EventSource): 229 | """ 230 | An asynchronous TCP server. 231 | 232 | Emits: 233 | - connect (tcp_conn): upon connection 234 | 235 | To start listening: 236 | 237 | > s = TcpServer(host, port) 238 | > s.on('connect', conn_handler) 239 | 240 | conn_handler is called every time a new client connects. 241 | """ 242 | 243 | def __init__( 244 | self, 245 | host: bytes, 246 | port: int, 247 | sock: Optional[socket.socket] = None, 248 | loop: Optional[LoopBase] = None, 249 | ) -> None: 250 | EventSource.__init__(self, loop) 251 | self.host = host 252 | self.port = port 253 | self.sock: Optional[socket.socket] = sock or server_listen(host, port) 254 | self.on("fd_readable", self.handle_accept) 255 | # self.sock is guaranteed to be a socket here because of server_listen, 256 | # but can become None later. 257 | if self.sock: 258 | self.register_fd(self.sock.fileno(), "fd_readable") 259 | schedule(0, self.emit, "start") 260 | self.active_connections: Set[TcpConnection] = set() 261 | self._shutting_down_gracefully = False 262 | 263 | def handle_accept(self) -> None: 264 | if not self.sock: 265 | return 266 | try: 267 | conn, _ = self.sock.accept() 268 | except (TypeError, IndexError): 269 | # sometimes accept() returns None if we have 270 | # multiple processes listening 271 | return 272 | conn.setblocking(False) 273 | try: 274 | tcp_conn = TcpConnection( 275 | conn, (self.host.decode("idna"), self.port), self.loop 276 | ) 277 | except FileNotFoundError: 278 | return # Connection closed in the meantime 279 | self.active_connections.add(tcp_conn) 280 | tcp_conn.on("disconnect", lambda: self._conn_closed(tcp_conn)) 281 | self.emit("connect", tcp_conn) 282 | 283 | def _conn_closed(self, conn: TcpConnection) -> None: 284 | self.active_connections.discard(conn) 285 | if self._shutting_down_gracefully and not self.active_connections: 286 | self.emit("stop") 287 | 288 | def graceful_shutdown(self) -> None: 289 | """ 290 | Stop accepting requests and wait for active connections to close. 291 | 292 | Emits "stop" when all connections are closed. 293 | """ 294 | self.remove_listeners("fd_readable") 295 | if self.sock: 296 | self.sock.close() 297 | self.sock = None 298 | self._shutting_down_gracefully = True 299 | if not self.active_connections: 300 | self.emit("stop") 301 | 302 | def shutdown(self) -> None: 303 | "Stop accepting requests and close the listening socket." 304 | self.remove_listeners("fd_readable") 305 | if self.sock: 306 | self.sock.close() 307 | self.sock = None 308 | self.emit("stop") 309 | 310 | 311 | def server_listen( 312 | host: bytes, port: int, backlog: Optional[int] = None 313 | ) -> socket.socket: 314 | "Return a socket listening to host:port." 315 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 316 | sock.setblocking(False) 317 | sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 318 | sock.bind((host, port)) 319 | sock.listen(backlog or socket.SOMAXCONN) 320 | return sock 321 | 322 | 323 | class TcpClient(EventSource): 324 | """ 325 | An asynchronous TCP client. 326 | 327 | Emits: 328 | - connect (tcp_conn): upon connection 329 | - connect_error (err_type, err_id, err_str): if there's a problem 330 | before getting a connection. err_type is 'socket' or 331 | 'gai'; err_id is the specific error encountered, and 332 | err_str is its textual description. 333 | 334 | To connect to a server: 335 | 336 | > c = TcpClient() 337 | > c.on('connect', conn_handler) 338 | > c.on('connect_error', error_handler) 339 | > c.connect(address) 340 | 341 | conn_handler will be called with the tcp_conn as the argument 342 | when the connection is made. 343 | """ 344 | 345 | def __init__(self, loop: Optional[LoopBase] = None) -> None: 346 | EventSource.__init__(self, loop) 347 | self.hostname: Optional[bytes] = None 348 | self.address: Optional[Address] = None 349 | self.sock: Optional[socket.socket] = None 350 | self.check_ip: Optional[Callable[[str], bool]] = None 351 | self._timeout_ev: Optional[ScheduledEvent] = None 352 | self._error_sent = False 353 | 354 | def connect( 355 | self, host: bytes, port: int, connect_timeout: Optional[float] = None 356 | ) -> None: 357 | """ 358 | Connect to an IPv4 host/port. Does not work with IPv6; see connect_dns(). 359 | """ 360 | dns_result = ( 361 | socket.AF_INET, 362 | socket.SOCK_STREAM, 363 | 6, 364 | "", 365 | (host.decode("idna"), port), 366 | ) 367 | self.connect_dns(host, dns_result, connect_timeout) 368 | 369 | def connect_dns( 370 | self, 371 | hostname: bytes, 372 | dns_result: DnsResult, 373 | connect_timeout: Optional[float] = None, 374 | ) -> None: 375 | """ 376 | Connect to DnsResult (with an optional connect timeout) 377 | and emit 'connect' when connected, or 'connect_error' in 378 | the case of an error. 379 | """ 380 | self.hostname = hostname 381 | family = dns_result[0] 382 | self.address = dns_result[4] 383 | if connect_timeout: 384 | self._timeout_ev = self.loop.schedule( 385 | connect_timeout, 386 | self.handle_socket_error, 387 | socket.error(errno.ETIMEDOUT, os.strerror(errno.ETIMEDOUT)), 388 | ) 389 | 390 | if callable(self.check_ip): 391 | if not self.check_ip(self.address[0]): 392 | self.handle_conn_error("access", 0, "IP Check failed") 393 | return 394 | 395 | self.sock = socket.socket(family, socket.SOCK_STREAM) 396 | self.sock.setblocking(False) 397 | self.register_fd(self.sock.fileno()) 398 | self.once("fd_error", self.handle_fd_error) 399 | self.event_add("fd_error") 400 | self.once("fd_writable", self.handle_connect) 401 | self.event_add("fd_writable") 402 | try: 403 | err = self.sock.connect_ex(self.address) 404 | except socket.error as why: 405 | self.handle_socket_error(why) 406 | return 407 | if err != errno.EINPROGRESS: 408 | self.handle_socket_error(socket.error(err, os.strerror(err))) 409 | return 410 | 411 | def handle_connect(self) -> None: 412 | self.unregister_fd() 413 | if self._timeout_ev: 414 | self._timeout_ev.delete() 415 | if self._error_sent: 416 | return 417 | assert self.sock, "Socket not found in handle_connect" 418 | err = self.sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) 419 | if err: 420 | self.handle_socket_error(socket.error(err, os.strerror(err))) 421 | else: 422 | assert self.address, "address not found in handle_connect" 423 | tcp_conn = TcpConnection(self.sock, self.address, self.loop) 424 | self.emit("connect", tcp_conn) 425 | 426 | def handle_fd_error(self) -> None: 427 | assert self.sock, "Socket not found in handle_fd_error" 428 | try: 429 | err_id = self.sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) 430 | except OSError: 431 | err_id = 54 432 | err_str = os.strerror(err_id) 433 | self.handle_conn_error("socket", err_id, err_str) 434 | 435 | def handle_socket_error(self, why: Exception, err_type: str = "socket") -> None: 436 | err_id = why.args[0] 437 | err_str = why.args[1] 438 | self.handle_conn_error(err_type, err_id, err_str) 439 | 440 | def handle_conn_error( 441 | self, err_type: str, err_id: int, err_str: str, close: bool = True 442 | ) -> None: 443 | """ 444 | Handle a connect error. 445 | """ 446 | if self._timeout_ev: 447 | self._timeout_ev.delete() 448 | if self._error_sent: 449 | return 450 | self._error_sent = True 451 | self.unregister_fd() 452 | self.emit("connect_error", err_type, err_id, err_str) 453 | if close and self.sock: 454 | self.sock.close() 455 | 456 | 457 | if __name__ == "__main__": 458 | # quick demo server 459 | from thor.loop import run, stop 460 | 461 | server = TcpServer(b"localhost", int(sys.argv[-1])) 462 | 463 | def handle_conn(conn: TcpConnection) -> None: 464 | conn.pause(False) 465 | 466 | def echo(chunk: bytes) -> None: 467 | if chunk.strip().lower() in ["quit", "stop"]: 468 | stop() 469 | else: 470 | conn.write(b"-> %s" % chunk) 471 | 472 | conn.on("data", echo) 473 | 474 | server.on("connect", handle_conn) 475 | run() 476 | -------------------------------------------------------------------------------- /test/test_http_parser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import sys 5 | import unittest 6 | 7 | from framework import DummyHttpParser 8 | from thor.http.common import Delimiters 9 | 10 | import thor.http.error as error 11 | 12 | 13 | class TestHttpParser(unittest.TestCase): 14 | def setUp(self): 15 | self.parser = DummyHttpParser() 16 | 17 | def checkSingleMsg(self, inputs, body, expected_err=None, close=False): 18 | """ 19 | Check a single HTTP message. 20 | """ 21 | assert type(inputs) == type([]) 22 | for chunk in inputs: 23 | self.parser.handle_input(chunk % {b"body": body, b"body_len": len(body)}) 24 | states = self.parser.test_states 25 | 26 | if not expected_err: 27 | self.assertFalse("ERROR" in states, self.parser.test_err) 28 | self.assertTrue(states.count("START") == 1, states) 29 | self.assertTrue(states.index("START") < states.index("BODY")) 30 | if close: 31 | self.assertEqual(self.parser._input_delimit, Delimiters.CLOSE) 32 | else: 33 | self.assertTrue(states.index("END") + 1 == len(states)) 34 | self.assertEqual( 35 | body, 36 | self.parser.test_body, 37 | f"{body[:20]} not equal to {self.parser.test_body[:20]}", 38 | ) 39 | else: 40 | self.assertTrue("ERROR" in states, states) 41 | self.assertEqual(self.parser.test_err.__class__, expected_err) 42 | 43 | def checkMultiMsg(self, inputs, body, count): 44 | """ 45 | Check pipelined messages. Assumes the same body for each (for now). 46 | """ 47 | for chunk in inputs: 48 | self.parser.handle_input(chunk % {b"body": body, b"body_len": len(body)}) 49 | states = self.parser.test_states 50 | self.assertFalse("ERROR" in self.parser.test_states, self.parser.test_err) 51 | self.parser.check(self, {"states": ["START", "BODY", "END"] * count}) 52 | 53 | def test_hdrs(self): 54 | body = b"12345678901234567890" 55 | self.checkSingleMsg( 56 | [ 57 | b"""\ 58 | http/1.1 200 OK 59 | Content-Type: text/plain 60 | Foo: bar 61 | Content-Length: %(body_len)i 62 | Foo: baz, bam 63 | 64 | %(body)s""" 65 | ], 66 | body, 67 | ) 68 | self.parser.check( 69 | self, 70 | { 71 | "hdrs": [ 72 | (b"Content-Type", b" text/plain"), 73 | (b"Foo", b" bar"), 74 | (b"Content-Length", b" %i" % len(body)), 75 | (b"Foo", b" baz, bam"), 76 | ] 77 | }, 78 | ) 79 | 80 | def test_hdrs_nocolon(self): 81 | body = b"12345678901234567890" 82 | self.checkSingleMsg( 83 | [ 84 | b"""\ 85 | http/1.1 200 OK 86 | Content-Type: text/plain 87 | Foo bar 88 | Content-Length: %(body_len)i 89 | 90 | %(body)s""" 91 | ], 92 | body, 93 | ) 94 | 95 | def test_hdr_case(self): 96 | body = b"12345678901234567890" 97 | self.checkSingleMsg( 98 | [ 99 | b"""\ 100 | http/1.1 200 OK 101 | Content-Type: text/plain 102 | content-LENGTH: %(body_len)i 103 | 104 | %(body)s""" 105 | ], 106 | body, 107 | ) 108 | 109 | def test_hdrs_whitespace_before_colon(self): 110 | body = b"lorum ipsum whatever goes after that." 111 | self.checkSingleMsg( 112 | [ 113 | b"""\ 114 | HTTP/1.1 200 OK 115 | Content-Type: text/plain 116 | Content-Length : %(body_len)i 117 | 118 | %(body)s""" 119 | ], 120 | body, 121 | error.HeaderSpaceError, 122 | ) 123 | 124 | def test_hdrs_fold(self): 125 | body = b"lorum ipsum whatever goes after that." 126 | self.checkSingleMsg( 127 | [ 128 | b"""\ 129 | HTTP/1.1 200 OK 130 | Content-Type: text/plain 131 | Foo: bar 132 | baz 133 | Content-Length: %(body_len)i 134 | 135 | %(body)s""" 136 | ], 137 | body, 138 | ) 139 | foo_val = [v for k, v in self.parser.test_hdrs if k == b"Foo"][-1] 140 | self.assertEqual(foo_val, b" bar baz") 141 | headers = [k for k, v in self.parser.test_hdrs] 142 | self.assertEqual(headers, [b"Content-Type", b"Foo", b"Content-Length"]) 143 | 144 | def test_hdrs_noname(self): 145 | body = b"lorum ipsum whatever goes after that." 146 | self.checkSingleMsg( 147 | [ 148 | b"""\ 149 | HTTP/1.1 200 OK 150 | Content-Type: text/plain 151 | : bar 152 | Content-Length: %(body_len)i 153 | 154 | %(body)s""" 155 | ], 156 | body, 157 | ) 158 | headers = [k for k, v in self.parser.test_hdrs] 159 | self.assertEqual(headers, [b"Content-Type", b"", b"Content-Length"]) 160 | 161 | def test_hdrs_utf8(self): 162 | body = b"lorum ipsum whatever goes after that." 163 | self.checkSingleMsg( 164 | [ 165 | """\ 166 | HTTP/1.1 200 OK 167 | Content-Type: text/plain 168 | Foo: ედუარდ შევარდნაძე 169 | Content-Length: %(body_len)i 170 | 171 | %(body)s""".encode( 172 | "utf-8" 173 | ) 174 | ], 175 | body, 176 | ) 177 | foo_val = [v for k, v in self.parser.test_hdrs if k == b"Foo"][-1] 178 | self.assertEqual(foo_val.decode("utf-8"), " ედუარდ შევარდნაძე") 179 | 180 | def test_hdrs_null(self): 181 | body = b"lorum ipsum whatever goes after that." 182 | self.checkSingleMsg( 183 | [ 184 | """\ 185 | HTTP/1.1 200 OK 186 | Content-Type: text/plain 187 | Foo: \0 188 | Content-Length: %(body_len)i 189 | 190 | %(body)s""".encode( 191 | "utf-8" 192 | ) 193 | ], 194 | body, 195 | ) 196 | foo_val = [v for k, v in self.parser.test_hdrs if k == b"Foo"][-1] 197 | self.assertEqual(foo_val, b" \0") 198 | 199 | def test_cl_delimit_11(self): 200 | body = b"lorum ipsum whatever goes after that." 201 | self.checkSingleMsg( 202 | [ 203 | b"""\ 204 | HTTP/1.1 200 OK 205 | Content-Type: text/plain 206 | Content-Length: %(body_len)i 207 | 208 | %(body)s""" 209 | ], 210 | body, 211 | ) 212 | 213 | def test_cl_delimit_10(self): 214 | body = b"abcdefghijklmnopqrstuvwxyz" 215 | self.checkSingleMsg( 216 | [ 217 | b"""\ 218 | HTTP/1.0 200 OK 219 | Content-Type: text/plain 220 | Content-Length: %(body_len)i 221 | 222 | %(body)s""" 223 | ], 224 | body, 225 | ) 226 | 227 | def test_close_delimit(self): 228 | body = b"abcdefghijklmnopqrstuvwxyz" 229 | self.checkSingleMsg( 230 | [ 231 | b"""\ 232 | HTTP/1.0 200 OK 233 | Content-Type: text/plain 234 | 235 | %(body)s""" 236 | ], 237 | body, 238 | close=True, 239 | ) 240 | 241 | def test_extra_line(self): 242 | body = b"lorum ipsum whatever goes after that." 243 | self.checkSingleMsg( 244 | [ 245 | b"""\ 246 | 247 | HTTP/1.1 200 OK 248 | Content-Type: text/plain 249 | Content-Length: %(body_len)i 250 | 251 | %(body)s""" 252 | ], 253 | body, 254 | ) 255 | 256 | def test_extra_lines(self): 257 | body = b"lorum ipsum whatever goes after that." 258 | self.checkSingleMsg( 259 | [ 260 | b"""\ 261 | 262 | 263 | 264 | HTTP/1.1 200 OK 265 | Content-Type: text/plain 266 | Content-Length: %(body_len)i 267 | 268 | %(body)s""" 269 | ], 270 | body, 271 | ) 272 | 273 | def test_telnet_client(self): 274 | body = "lorum ipsum whatever goes after that." 275 | self.checkSingleMsg( 276 | [ 277 | a.encode("ascii") 278 | for a in f""" 279 | 280 | 281 | HTTP/1.1 200 OK 282 | Content-Type: text/plain 283 | Content-Length: {len(body)} 284 | 285 | {body}""" 286 | ], 287 | body.encode("ascii"), 288 | ) 289 | 290 | def test_naughty_first_header(self): 291 | body = b"lorum ipsum whatever goes after that." 292 | self.checkSingleMsg( 293 | [ 294 | b"""\ 295 | HTTP/1.1 200 OK 296 | Content-Type: text/plain 297 | Content-Length: %(body_len)i 298 | 299 | %(body)s""" 300 | ], 301 | body, 302 | error.TopLineSpaceError, 303 | ) 304 | 305 | def test_cl_header_case(self): 306 | body = b"12345678901234567890" 307 | self.checkSingleMsg( 308 | [ 309 | b"""\ 310 | HTTP/1.1 200 OK 311 | Content-Type: text/plain 312 | content-LENGTH: %(body_len)i 313 | 314 | %(body)s""" 315 | ], 316 | body, 317 | ) 318 | 319 | def test_chunk_delimit(self): 320 | body = b"aaabbbcccdddeeefffggghhhiii" 321 | self.checkSingleMsg( 322 | [ 323 | b"""\ 324 | HTTP/1.1 200 OK 325 | Content-Type: text/plain 326 | Transfer-Encoding: chunked 327 | 328 | %(body_len)x\r 329 | %(body)s\r 330 | 0\r 331 | \r 332 | """ 333 | ], 334 | body, 335 | ) 336 | 337 | def test_chunk_exact(self): 338 | body = b"aaabbbcccdddeeefffggghhhiii" 339 | self.checkSingleMsg( 340 | [ 341 | b"""\ 342 | HTTP/1.1 200 OK 343 | Content-Type: text/plain 344 | Transfer-Encoding: chunked 345 | 346 | """, 347 | b"""\ 348 | %(body_len)x\r 349 | %(body)s\r 350 | """, 351 | b"""\ 352 | 0\r 353 | \r 354 | """, 355 | ], 356 | body, 357 | ) 358 | 359 | def test_chunk_split(self): 360 | body = b"aaabbbcccdddeeefffggghhhiii" 361 | self.checkSingleMsg( 362 | [ 363 | b"""\ 364 | HTTP/1.1 200 OK 365 | Content-Type: text/plain 366 | Transfer-Encoding: chunked 367 | 368 | """, 369 | b"""\ 370 | %(body_len)x\r 371 | %(body)s\r 372 | 0""", 373 | b"""\ 374 | \r 375 | Foo: bar\r 376 | \r 377 | """, 378 | ], 379 | body, 380 | ) 381 | 382 | def test_chunk_exact_offset(self): 383 | body = b"aaabbbcccdddeeefffggghhhiii" 384 | self.checkSingleMsg( 385 | [ 386 | b"""\ 387 | HTTP/1.1 200 OK 388 | Content-Type: text/plain 389 | Transfer-Encoding: chunked 390 | 391 | """, 392 | b"""\ 393 | %(body_len)x\r 394 | %(body)s""", 395 | b"""\r 396 | 0\r 397 | \r 398 | """, 399 | ], 400 | body, 401 | ) 402 | 403 | def test_chunk_more(self): 404 | body = b"1234567890" 405 | self.checkSingleMsg( 406 | [ 407 | b"""\ 408 | HTTP/1.1 200 OK 409 | Content-Type: text/plain 410 | Transfer-Encoding: chunked 411 | 412 | """, 413 | b"""\ 414 | %(body_len)x\r 415 | %(body)s\r 416 | %(body_len)x\r 417 | %(body)s\r 418 | 0\r 419 | \r 420 | """ 421 | % {b"body": body, b"body_len": len(body)}, 422 | ], 423 | body * 2, 424 | ) 425 | 426 | def test_transfer_case(self): 427 | body = b"aaabbbcccdddeeefffggghhhiii" 428 | self.checkSingleMsg( 429 | [ 430 | b"""\ 431 | HTTP/1.1 200 OK 432 | Content-Type: text/plain 433 | Transfer-Encoding: cHuNkEd 434 | 435 | %(body_len)x\r 436 | %(body)s\r 437 | 0\r 438 | \r 439 | """ 440 | ], 441 | body, 442 | ) 443 | 444 | def test_big_chunk(self): 445 | body = b"aaabbbcccdddeeefffggghhhiii" * 1000000 446 | self.checkSingleMsg( 447 | [ 448 | b"""\ 449 | HTTP/1.1 200 OK 450 | Content-Type: text/plain 451 | Transfer-Encoding: chunked 452 | 453 | %(body_len)x\r 454 | %(body)s\r 455 | 0\r 456 | \r 457 | """ 458 | ], 459 | body, 460 | ) 461 | 462 | def xxxtest_small_chunks(self): 463 | num_chunks = 10000 464 | body = b"a" * num_chunks 465 | inputs = [ 466 | b"""\ 467 | HTTP/1.1 200 OK 468 | Content-Type: text/plain 469 | Transfer-Encoding: chunked 470 | 471 | """ 472 | ] 473 | for i in range(num_chunks): 474 | inputs.append( 475 | b"""\ 476 | 1\r 477 | a\r 478 | """ 479 | ) 480 | inputs.append( 481 | b"""\ 482 | 0\r 483 | \r 484 | """ 485 | ) 486 | self.checkSingleMsg(inputs, body) 487 | 488 | def test_split_chunk(self): 489 | body = b"abcdefg123456" 490 | self.checkSingleMsg( 491 | [ 492 | b"""\ 493 | HTTP/1.1 200 OK 494 | Content-Type: text/plain 495 | Transfer-Encoding: chunked 496 | 497 | %(body_len)x\r 498 | abcdefg""", 499 | b"""\ 500 | 123456\r 501 | 0\r 502 | \r 503 | """, 504 | ], 505 | body, 506 | ) 507 | 508 | def test_split_chunk_length(self): 509 | body = b"do re mi so fa la ti do" 510 | self.checkSingleMsg( 511 | [ 512 | b"""\ 513 | HTTP/1.1 200 OK 514 | Content-Type: text/plain 515 | Transfer-Encoding: chunked 516 | 517 | %(body_len)x""", 518 | b"""\ 519 | \r 520 | %(body)s\r 521 | 0\r 522 | \r 523 | """, 524 | ], 525 | body, 526 | ) 527 | 528 | def test_chunk_bad_syntax(self): 529 | body = b"abc123def456ghi789" 530 | self.checkSingleMsg( 531 | [ 532 | b"""\ 533 | HTTP/1.1 200 OK 534 | Content-Type: text/plain 535 | Transfer-Encoding: chunked 536 | 537 | ZZZZ\r 538 | %(body)s\r 539 | 0\r 540 | \r 541 | """ 542 | ], 543 | body, 544 | error.ChunkError, 545 | ) 546 | 547 | def test_chunk_nonfinal(self): 548 | body = b"abc123def456ghi789" 549 | self.checkSingleMsg( 550 | [ 551 | b"""\ 552 | HTTP/1.1 200 OK 553 | Content-Type: text/plain 554 | Transfer-Encoding: chunked, foo 555 | 556 | %(body)s""" 557 | ], 558 | body, 559 | close=True, 560 | ) 561 | 562 | def test_cl_dup(self): 563 | body = b"abc123def456ghi789" 564 | self.checkSingleMsg( 565 | [ 566 | b"""\ 567 | HTTP/1.1 200 OK 568 | Content-Type: text/plain 569 | Content-Length: %(body_len)i 570 | Content-Length: %(body_len)i 571 | 572 | %(body)s""" 573 | ], 574 | body, 575 | ) 576 | 577 | def test_cl_conflict(self): 578 | body = b"abc123def456ghi789" 579 | self.checkSingleMsg( 580 | [ 581 | b"""\ 582 | HTTP/1.1 200 OK 583 | Content-Type: text/plain 584 | Content-Length: 2 585 | Content-Length: %(body_len)i 586 | 587 | %(body)s""" 588 | ], 589 | body, 590 | error.DuplicateCLError, 591 | ) 592 | 593 | def test_cl_bad_syntax(self): 594 | body = b"abc123def456ghi789" 595 | self.checkSingleMsg( 596 | [ 597 | b"""\ 598 | HTTP/1.1 200 OK 599 | Content-Type: text/plain 600 | Content-Length: 2abc 601 | 602 | %(body)s""" 603 | ], 604 | body, 605 | error.MalformedCLError, 606 | ) 607 | 608 | def test_chunk_ext(self): 609 | body = b"abc123def456ghi789" 610 | self.checkSingleMsg( 611 | [ 612 | b"""\ 613 | HTTP/1.1 200 OK 614 | Content-Type: text/plain 615 | Transfer-Encoding: chunked 616 | 617 | %(body_len)x; myext=foobarbaz\r 618 | %(body)s\r 619 | 0\r 620 | \r 621 | """ 622 | ], 623 | body, 624 | ) 625 | 626 | def test_trailers(self): 627 | body = b"abc123def456ghi789" 628 | self.checkSingleMsg( 629 | [ 630 | b"""\ 631 | HTTP/1.1 200 OK 632 | Content-Type: text/plain 633 | Transfer-Encoding: chunked 634 | 635 | %(body_len)x\r 636 | %(body)s\r 637 | 0\r 638 | Foo: bar 639 | Baz: 1 640 | \r 641 | """ 642 | ], 643 | body, 644 | ) 645 | self.assertEqual( 646 | self.parser.test_trailers, [(b"Foo", b" bar"), (b"Baz", b" 1")] 647 | ) 648 | 649 | def test_pipeline_chunked(self): 650 | body = b"abc123def456ghi789" 651 | self.checkMultiMsg( 652 | [ 653 | b"""\ 654 | HTTP/1.1 200 OK 655 | Content-Type: text/plain 656 | Transfer-Encoding: chunked 657 | 658 | %(body_len)x\r 659 | %(body)s\r 660 | 0\r 661 | \r 662 | HTTP/1.1 404 Not Found 663 | Content-Type: text/plain 664 | Transfer-Encoding: chunked 665 | 666 | %(body_len)x\r 667 | %(body)s\r 668 | 0\r 669 | \r 670 | """ 671 | ], 672 | body, 673 | 2, 674 | ) 675 | 676 | def test_pipeline_cl(self): 677 | body = b"abc123def456ghi789" 678 | self.checkMultiMsg( 679 | [ 680 | b"""\ 681 | HTTP/1.1 200 OK 682 | Content-Type: text/plain 683 | Content-Length: %(body_len)i 684 | 685 | %(body)sHTTP/1.1 404 Not Found 686 | Content-Type: text/plain 687 | Content-Length: %(body_len)i 688 | 689 | %(body)s""" 690 | ], 691 | body, 692 | 2, 693 | ) 694 | 695 | 696 | # def test_nobody_delimit(self): 697 | # def test_pipeline_nobody(self): 698 | # def test_chunked_then_length(self): 699 | # def test_length_then_chunked(self): 700 | 701 | 702 | if __name__ == "__main__": 703 | unittest.main() 704 | -------------------------------------------------------------------------------- /thor/loop.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Asynchronous event loops 5 | 6 | This is a generic library for building asynchronous event loops, using 7 | Python's built-in poll / epoll / kqueue support. 8 | """ 9 | 10 | import contextvars 11 | from abc import ABCMeta, abstractmethod 12 | import cProfile 13 | import errno 14 | from functools import partial 15 | import select 16 | import time as systime 17 | from typing import ( 18 | Callable, 19 | List, 20 | Dict, 21 | Optional, 22 | Set, 23 | Iterable, 24 | Tuple, 25 | Any, 26 | ) 27 | 28 | from thor.events import EventEmitter 29 | 30 | 31 | __all__ = ["run", "stop", "schedule", "time"] 32 | 33 | 34 | class EventSource(EventEmitter): 35 | """ 36 | Base class for objects that the loop will direct interesting 37 | events to. 38 | 39 | An instance should map to one thing with an interesting file 40 | descriptor, registered with register_fd. 41 | """ 42 | 43 | def __init__(self, loop: Optional["LoopBase"] = None) -> None: 44 | EventEmitter.__init__(self) 45 | self.loop = loop or _loop 46 | self._interesting_events: Set[str] = set() 47 | self._fd: int = -1 48 | 49 | def register_fd(self, fd: int, event: Optional[str] = None) -> None: 50 | """ 51 | Register myself with the loop using file descriptor fd. 52 | If event is specified, start emitting it. 53 | """ 54 | self._fd = fd 55 | self.loop.register_fd(self._fd, [], self) 56 | if event: 57 | self.event_add(event) 58 | 59 | def unregister_fd(self) -> None: 60 | "Unregister myself from the loop." 61 | if self._fd >= 0: 62 | self.loop.unregister_fd(self._fd) 63 | self._fd = -1 64 | 65 | def event_add(self, event: str) -> None: 66 | "Start emitting the given event." 67 | if event not in self._interesting_events: 68 | self._interesting_events.add(event) 69 | self.loop.event_add(self._fd, event) 70 | 71 | def event_del(self, event: str) -> None: 72 | "Stop emitting the given event." 73 | if event in self._interesting_events: 74 | self._interesting_events.remove(event) 75 | self.loop.event_del(self._fd, event) 76 | 77 | def interesting_events(self) -> Set[str]: 78 | return self._interesting_events 79 | 80 | 81 | class LoopBase(EventEmitter, metaclass=ABCMeta): 82 | """ 83 | Base class for async loops. 84 | """ 85 | 86 | _event_types: Dict[int, str] = {} # map of event types to names; override. 87 | 88 | def __init__(self, precision: Optional[float] = None) -> None: 89 | EventEmitter.__init__(self) 90 | self.precision = precision or 0.1 # of running scheduled queue (secs) 91 | self.running = False # whether or not the loop is running (read-only) 92 | self.debug = False 93 | self.__profiler: cProfile.Profile 94 | self.__sched_events: List[Tuple[float, Callable]] = [] 95 | self._fd_targets: Dict[int, EventSource] = {} 96 | self.__last_event_check: float = 0.0 97 | self._eventlookup = {v: k for (k, v) in self._event_types.items()} 98 | self.__event_cache: Dict[int, Set[str]] = {} 99 | 100 | def __repr__(self) -> str: 101 | name = self.__class__.__name__ 102 | is_running = "running" if self.running else "not-running" 103 | events = len(self.__sched_events) 104 | targets = len(self._fd_targets) 105 | return f"<{name} - {is_running}, {events} sched_events, {targets} fd_targets>" 106 | 107 | def run(self) -> None: 108 | "Start the loop." 109 | self.running = True 110 | self.emit("start") 111 | if self.debug: 112 | self.__profiler = cProfile.Profile() 113 | while self.running: 114 | if self.debug: 115 | fd_start = systime.monotonic() 116 | self.__profiler.enable() 117 | self._run_fd_events() 118 | self.__profiler.disable() 119 | delay = systime.monotonic() - fd_start 120 | if delay > self.precision * 2: 121 | self.debug_out(f"long fd delay ({delay:.2f})", self.__profiler) 122 | else: 123 | self._run_fd_events() 124 | # find scheduled events 125 | if systime.monotonic() - self.__last_event_check >= self.precision: 126 | self._run_scheduled_events() 127 | 128 | def debug_out(self, message: str, profile: Optional[cProfile.Profile]) -> None: 129 | "Output a debug message and profile. Should be overridden." 130 | 131 | def registered_fd_handlers(self) -> List[EventSource]: 132 | "Returns a list of registered fd EventSources." 133 | return list(self._fd_targets.values()) 134 | 135 | @abstractmethod 136 | def _run_fd_events(self) -> None: 137 | "Run loop-specific FD events." 138 | raise NotImplementedError 139 | 140 | def _run_scheduled_events(self) -> None: 141 | "Run scheduled events." 142 | if self.debug: 143 | if len(self.__sched_events) > 500: 144 | self.debug_out(f"{len(self.__sched_events)} events scheduled", None) 145 | self.__last_event_check = systime.monotonic() 146 | while self.__sched_events: 147 | when, what = self.__sched_events[0] 148 | if self.running and when <= self.__last_event_check: 149 | self.__sched_events.pop(0) 150 | if self.debug: 151 | ev_start = systime.monotonic() 152 | self.__profiler.enable() 153 | what() 154 | self.__profiler.disable() 155 | delay = systime.monotonic() - ev_start 156 | if delay > self.precision * 2: 157 | name = getattr(what, "__name__", str(what)) 158 | self.debug_out( 159 | f"long scheduled event delay ({delay:.2f}): {name}", 160 | self.__profiler, 161 | ) 162 | else: 163 | what() 164 | else: 165 | break 166 | 167 | def scheduled_events(self) -> List[Tuple[float, Callable]]: 168 | """ 169 | Return a list of (delay, callback) tuples for currently scheduled events. 170 | Delay is in seconds, measured from call time. 171 | """ 172 | now = systime.monotonic() 173 | return [(when - now, what) for when, what in self.__sched_events] 174 | 175 | def stop(self) -> None: 176 | "Stop the loop and unregister all fds." 177 | self.__sched_events = [] 178 | self.running = False 179 | for fd in list(self._fd_targets): 180 | self.unregister_fd(fd) 181 | self.emit("stop") 182 | 183 | @abstractmethod 184 | def register_fd(self, fd: int, events: List[str], target: EventSource) -> None: 185 | "emit events on target when they occur on fd." 186 | raise NotImplementedError 187 | 188 | @abstractmethod 189 | def unregister_fd(self, fd: int) -> None: 190 | "Stop emitting events from fd." 191 | raise NotImplementedError 192 | 193 | def fd_count(self) -> int: 194 | "Return how many FDs are currently monitored by the loop." 195 | return len(self._fd_targets) 196 | 197 | @abstractmethod 198 | def event_add(self, fd: int, event: str) -> None: 199 | "Start emitting event for fd." 200 | raise NotImplementedError 201 | 202 | @abstractmethod 203 | def event_del(self, fd: int, event: str) -> None: 204 | "Stop emitting event for fd" 205 | raise NotImplementedError 206 | 207 | def _fd_event(self, event: str, fd: int) -> None: 208 | "An event has occured on an fd." 209 | if fd in self._fd_targets: 210 | self._fd_targets[fd].emit(event) 211 | 212 | def time(self) -> float: 213 | "Return the current time (deprecated)." 214 | return systime.time() 215 | 216 | def schedule( 217 | self, delta: float, callback: Callable, *args: Any 218 | ) -> "ScheduledEvent": 219 | """ 220 | Schedule callable callback to be run in delta seconds with *args. 221 | 222 | Returns an object which can be used to later remove the event, by 223 | calling its delete() method. 224 | """ 225 | 226 | ctx = contextvars.copy_context() 227 | 228 | def ctx_callback(*args: Any, **kwargs: Any) -> Any: 229 | return ctx.run(callback, *args, **kwargs) 230 | 231 | cb = partial(ctx_callback, *args) 232 | new_event = (systime.monotonic() + delta, cb) 233 | events = self.__sched_events 234 | self._insort(events, new_event) 235 | if delta > self.precision: 236 | self._run_scheduled_events() 237 | return ScheduledEvent(self, new_event) 238 | 239 | def schedule_del(self, event: Tuple[float, Callable]) -> None: 240 | try: 241 | self.__sched_events.remove(event) 242 | except ValueError: # already gone 243 | pass 244 | 245 | @staticmethod 246 | def _insort(li: List, thing: Any, lo: int = 0, hi: Optional[int] = None) -> None: 247 | if lo < 0: 248 | raise ValueError("lo must be non-negative") 249 | if hi is None: 250 | hi = len(li) 251 | while lo < hi: 252 | mid = (lo + hi) // 2 253 | if thing[0] < li[mid][0]: 254 | hi = mid 255 | else: 256 | lo = mid + 1 257 | li.insert(lo, thing) 258 | 259 | def _eventmask(self, events: Iterable[str]) -> int: 260 | "Calculate the mask for a list of events." 261 | eventmask = 0 262 | for event in events: 263 | eventmask |= self._eventlookup.get(event, 0) 264 | return eventmask 265 | 266 | def _filter2events(self, evfilter: int) -> Set[str]: 267 | "Calculate the events implied by a given filter." 268 | if evfilter not in self.__event_cache: 269 | events = set() 270 | for et, ev in self._event_types.items(): 271 | if et & evfilter: 272 | events.add(ev) 273 | self.__event_cache[evfilter] = events 274 | return self.__event_cache[evfilter] 275 | 276 | 277 | class ScheduledEvent: 278 | """ 279 | Holds a scheduled event. 280 | """ 281 | 282 | def __init__(self, loop: LoopBase, event: Tuple[float, Callable]) -> None: 283 | self._loop = loop 284 | self._event = event 285 | self._deleted = False 286 | 287 | def delete(self) -> None: 288 | if not self._deleted: 289 | self._loop.schedule_del(self._event) 290 | self._deleted = True 291 | 292 | 293 | class PollLoop(LoopBase): 294 | """ 295 | A poll()-based async loop. 296 | """ 297 | 298 | def __init__(self, *args: Any) -> None: 299 | # pylint: disable=E1101 300 | self._event_types = { 301 | select.POLLIN: "fd_readable", # type: ignore[attr-defined] 302 | select.POLLOUT: "fd_writable", # type: ignore[attr-defined] 303 | select.POLLERR: "fd_error", # type: ignore[attr-defined] 304 | select.POLLHUP: "fd_close", # type: ignore[attr-defined] 305 | } 306 | 307 | LoopBase.__init__(self, *args) 308 | self._poll = select.poll() 309 | # pylint: enable=E1101 310 | 311 | def register_fd(self, fd: int, events: List[str], target: EventSource) -> None: 312 | self._fd_targets[fd] = target 313 | self._poll.register(fd, self._eventmask(events)) 314 | 315 | def unregister_fd(self, fd: int) -> None: 316 | self._poll.unregister(fd) 317 | del self._fd_targets[fd] 318 | 319 | def event_add(self, fd: int, event: str) -> None: 320 | try: 321 | eventmask = self._eventmask(self._fd_targets[fd].interesting_events()) 322 | except KeyError: 323 | return 324 | self._poll.register(fd, eventmask) 325 | 326 | def event_del(self, fd: int, event: str) -> None: 327 | try: 328 | eventmask = self._eventmask(self._fd_targets[fd].interesting_events()) 329 | except KeyError: 330 | return 331 | self._poll.register(fd, eventmask) 332 | 333 | def _run_fd_events(self) -> None: 334 | try: 335 | event_list = self._poll.poll(int(self.precision * 1000)) 336 | except (OSError, select.error) as why: 337 | if why.args[0] == errno.EINTR: 338 | return 339 | raise 340 | for fileno, eventmask in event_list: 341 | events = self._filter2events(eventmask) 342 | if "fd_readable" in events: 343 | self._fd_event("fd_readable", fileno) 344 | if "fd_writable" in events: 345 | self._fd_event("fd_writable", fileno) 346 | for event in events: 347 | if event not in ("fd_readable", "fd_writable"): 348 | self._fd_event(event, fileno) 349 | 350 | 351 | class EpollLoop(LoopBase): 352 | """ 353 | An epoll()-based async loop. Currently level-triggered. 354 | """ 355 | 356 | def __init__(self, *args: Any) -> None: 357 | # pylint: disable=E1101 358 | self._event_types = { 359 | select.EPOLLIN: "fd_readable", # type: ignore[attr-defined] 360 | select.EPOLLOUT: "fd_writable", # type: ignore[attr-defined] 361 | select.EPOLLRDHUP: "fd_close", # type: ignore[attr-defined] 362 | select.EPOLLHUP: "fd_close", # type: ignore[attr-defined] 363 | select.EPOLLERR: "fd_error", # type: ignore[attr-defined] 364 | } 365 | LoopBase.__init__(self, *args) 366 | self._epoll = select.epoll() # type: ignore[attr-defined] 367 | # pylint: enable=E1101 368 | 369 | def register_fd(self, fd: int, events: List[str], target: EventSource) -> None: 370 | eventmask = self._eventmask(events) 371 | if fd in self._fd_targets: 372 | self._epoll.modify(fd, eventmask) 373 | else: 374 | self._fd_targets[fd] = target 375 | self._epoll.register(fd, eventmask) 376 | 377 | def unregister_fd(self, fd: int) -> None: 378 | try: 379 | del self._fd_targets[fd] 380 | except KeyError: 381 | pass 382 | try: 383 | self._epoll.unregister(fd) 384 | except FileNotFoundError: 385 | return # already unregistered 386 | except OSError as why: 387 | if why.errno == errno.EBADF: 388 | return # already unregistered 389 | raise 390 | 391 | def event_add(self, fd: int, event: str) -> None: 392 | try: 393 | eventmask = self._eventmask(self._fd_targets[fd].interesting_events()) 394 | except KeyError: 395 | return 396 | self._epoll.modify(fd, eventmask) 397 | 398 | def event_del(self, fd: int, event: str) -> None: 399 | try: 400 | eventmask = self._eventmask(self._fd_targets[fd].interesting_events()) 401 | except KeyError: 402 | return # no longer interested 403 | self._epoll.modify(fd, eventmask) 404 | 405 | def _run_fd_events(self) -> None: 406 | try: 407 | event_list = self._epoll.poll(self.precision) 408 | except (OSError, select.error) as why: 409 | if why.args[0] == errno.EINTR: 410 | return 411 | raise 412 | for fileno, eventmask in event_list: 413 | events = self._filter2events(eventmask) 414 | if "fd_readable" in events: 415 | self._fd_event("fd_readable", fileno) 416 | if "fd_writable" in events: 417 | self._fd_event("fd_writable", fileno) 418 | for event in events: 419 | if event not in ("fd_readable", "fd_writable"): 420 | self._fd_event(event, fileno) 421 | 422 | 423 | class KqueueLoop(LoopBase): 424 | """ 425 | A kqueue()-based async loop. 426 | """ 427 | 428 | def __init__(self, *args: Any) -> None: 429 | # pylint: disable=E1101 430 | self._event_types = { 431 | select.KQ_FILTER_READ: "fd_readable", # type: ignore[attr-defined] 432 | select.KQ_FILTER_WRITE: "fd_writable", # type: ignore[attr-defined] 433 | } 434 | LoopBase.__init__(self, *args) 435 | self.max_ev = 50 # maximum number of events to pull from the queue 436 | self._kq = select.kqueue() # type: ignore[attr-defined] 437 | # pylint: enable=E1101 438 | 439 | def register_fd(self, fd: int, events: List[str], target: EventSource) -> None: 440 | self._fd_targets[fd] = target 441 | for event in events: 442 | self.event_add(fd, event) 443 | 444 | def unregister_fd(self, fd: int) -> None: 445 | try: 446 | obj = self._fd_targets[fd] 447 | except KeyError: 448 | return 449 | for event in list(obj.interesting_events()): 450 | obj.event_del(event) 451 | del self._fd_targets[fd] 452 | 453 | def event_add(self, fd: int, event: str) -> None: 454 | eventmask = self._eventmask([event]) 455 | if eventmask: 456 | ev = select.kevent( # type: ignore[attr-defined] 457 | fd, 458 | eventmask, 459 | select.KQ_EV_ADD | select.KQ_EV_ENABLE, # type: ignore[attr-defined] 460 | ) 461 | self._kq.control([ev], 0, 0) 462 | 463 | def event_del(self, fd: int, event: str) -> None: 464 | eventmask = self._eventmask([event]) 465 | if eventmask: 466 | ev = select.kevent( # type: ignore[attr-defined] 467 | fd, 468 | eventmask, 469 | select.KQ_EV_DELETE, # type: ignore[attr-defined] 470 | ) 471 | if ev: 472 | try: 473 | self._kq.control([ev], 0, 0) 474 | except FileNotFoundError: 475 | pass 476 | 477 | def _run_fd_events(self) -> None: 478 | try: 479 | events = self._kq.control([], self.max_ev, self.precision) 480 | except (OSError, select.error) as why: 481 | if why.args[0] == errno.EINTR: 482 | return 483 | raise 484 | for ev in events: 485 | fileno = int(ev.ident) 486 | if ev.flags & select.KQ_EV_ERROR: # type: ignore[attr-defined] 487 | self._fd_event("fd_error", fileno) 488 | continue 489 | if ev.flags & select.KQ_EV_EOF: # type: ignore[attr-defined] 490 | if ev.fflags: 491 | self._fd_event("fd_error", fileno) 492 | 493 | event_types = self._filter2events(ev.filter) 494 | for event_type in event_types: 495 | self._fd_event(event_type, fileno) 496 | 497 | if ev.flags & select.KQ_EV_EOF: # type: ignore[attr-defined] 498 | self._fd_event("fd_close", fileno) 499 | 500 | 501 | def make(precision: Optional[float] = None) -> LoopBase: 502 | """ 503 | Create and return a named loop that is suitable for the current system. If 504 | _precision_ is given, it indicates how often scheduled events will be run. 505 | 506 | Returned loop instances have all of the methods and instance variables 507 | that *thor.loop* has. 508 | """ 509 | loop: LoopBase 510 | if hasattr(select, "epoll"): 511 | loop = EpollLoop(precision) 512 | elif hasattr(select, "kqueue"): 513 | loop = KqueueLoop(precision) 514 | elif hasattr(select, "poll"): 515 | loop = PollLoop(precision) 516 | else: 517 | raise ImportError("What is this thing, a Windows box?") 518 | return loop 519 | 520 | 521 | _loop = make() # by default, just one big loop. 522 | run = _loop.run 523 | stop = _loop.stop 524 | schedule = _loop.schedule 525 | time = _loop.time 526 | --------------------------------------------------------------------------------