├── .github └── workflows │ ├── coverage.yml │ ├── integration.yml │ └── unit.yml ├── .gitignore ├── .readthedocs.yaml ├── DEVELOPERS.rst ├── LICENSE ├── Makefile ├── NEWS.rst ├── README.rst ├── docs ├── Makefile ├── README.rst ├── _static │ └── magic-peer-networking.svg ├── conf.py ├── fowl-forward-light.png ├── fowl-logo-0.png ├── frontend-protocol.rst ├── index.rst ├── logo.svg ├── make.bat ├── protocol.rst ├── releases.rst ├── requirements.txt └── usage.rst ├── fowl-interaction-screenshot-1000.png ├── integration ├── conftest.py ├── test_happy.py ├── test_human.py └── util.py ├── logo.svg ├── pyproject.toml ├── requirements-pinned.txt ├── src └── fowl │ ├── __init__.py │ ├── __main__.py │ ├── _proto.py │ ├── _tui.py │ ├── chicken.py │ ├── cli.py │ ├── daemon.py │ ├── messages.py │ ├── observer.py │ ├── policy.py │ ├── status.py │ ├── test │ ├── __init__.py │ ├── conftest.py │ ├── test_cli.py │ ├── test_commands.py │ ├── test_forward.py │ ├── test_policy.py │ └── util.py │ └── visual.py ├── testcase.py └── update-version.py /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | name: Coverage 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | coverage-unit-integration: 11 | strategy: 12 | matrix: 13 | os: 14 | - runs-on: ubuntu-latest 15 | python-version: 16 | - "3.12" 17 | runs-on: ${{ matrix.os.runs-on }} 18 | steps: 19 | - name: Check out source code 20 | uses: actions/checkout@v2 21 | with: 22 | # Get enough history for the tags we get next to be meaningful. 0 23 | # means all history. 24 | fetch-depth: "0" 25 | # Checkout head of the branch of the PR, or the exact revision 26 | # specified for non-PR builds. 27 | ref: "${{ github.event.pull_request.head.sha || github.sha }}" 28 | 29 | - name: Set up Python ${{ matrix.python-version }} 30 | uses: actions/setup-python@v2 31 | with: 32 | python-version: ${{ matrix.python-version }} 33 | 34 | - name: Install dependencies 35 | run: | 36 | python -m pip install --upgrade pip 37 | pip install --editable .[test] 38 | 39 | - name: Run Integration Tests 40 | run: coverage run --parallel -m pytest -s -v integration/ 41 | 42 | - name: Run Unit Tests 43 | run: coverage run --parallel -m pytest -s -v src/fowl/test 44 | 45 | - name: Combine Coverage 46 | run: coverage combine 47 | 48 | - name: Coverage graph 49 | shell: bash 50 | run: cuv graph 51 | continue-on-error: true 52 | 53 | - name: PR Coverage Graph 54 | shell: bash 55 | run: git diff origin/main..HEAD | cuv diff - 56 | continue-on-error: true 57 | 58 | - name: Coverage report 59 | shell: bash 60 | run: | 61 | git diff origin/main..HEAD > p 62 | cuv report p 63 | continue-on-error: true 64 | 65 | - name: Coverage summary 66 | shell: bash 67 | run: | 68 | git diff origin/main..HEAD > p 69 | echo "Coverage" > $GITHUB_STEP_SUMMARY 70 | echo "--------" >> $GITHUB_STEP_SUMMARY 71 | cuv report p >> $GITHUB_STEP_SUMMARY 72 | continue-on-error: true 73 | -------------------------------------------------------------------------------- /.github/workflows/integration.yml: -------------------------------------------------------------------------------- 1 | name: Linux Testing 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | integration-tests: 11 | strategy: 12 | matrix: 13 | os: 14 | - runs-on: ubuntu-latest 15 | python-version: 16 | - "3.10" 17 | - "3.11" 18 | - "3.12" 19 | runs-on: ${{ matrix.os.runs-on }} 20 | steps: 21 | - name: Check out source code 22 | uses: actions/checkout@v2 23 | with: 24 | # Get enough history for the tags we get next to be meaningful. 0 25 | # means all history. 26 | fetch-depth: "0" 27 | # Checkout head of the branch of the PR, or the exact revision 28 | # specified for non-PR builds. 29 | ref: "${{ github.event.pull_request.head.sha || github.sha }}" 30 | 31 | - name: Set up Python ${{ matrix.python-version }} 32 | uses: actions/setup-python@v2 33 | with: 34 | python-version: ${{ matrix.python-version }} 35 | 36 | - name: Install dependencies 37 | run: | 38 | python -m pip install --upgrade pip 39 | pip install --editable .[test] 40 | 41 | - name: Lint with ruff 42 | run: ruff check src/fowl 43 | 44 | - name: pytest 45 | run: pytest --cov=fowl -s -v integration/ 46 | 47 | - name: Coverage graph 48 | shell: bash 49 | run: cuv graph 50 | continue-on-error: true 51 | 52 | - name: Coverage report 53 | shell: bash 54 | run: | 55 | git diff origin/main..HEAD > p 56 | cuv report p 57 | continue-on-error: true 58 | -------------------------------------------------------------------------------- /.github/workflows/unit.yml: -------------------------------------------------------------------------------- 1 | name: Unit Testing 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | unit-tests: 11 | strategy: 12 | matrix: 13 | os: 14 | - runs-on: ubuntu-latest 15 | python-version: 16 | - "3.10" 17 | - "3.11" 18 | - "3.12" 19 | # - "3.13-dev" # "greenlet" doesn't build? (req'd by pytest_twisted) 20 | runs-on: ${{ matrix.os.runs-on }} 21 | steps: 22 | - name: Check out source code 23 | uses: actions/checkout@v2 24 | with: 25 | # Get enough history for the tags we get next to be meaningful. 0 26 | # means all history. 27 | fetch-depth: "0" 28 | # Checkout head of the branch of the PR, or the exact revision 29 | # specified for non-PR builds. 30 | ref: "${{ github.event.pull_request.head.sha || github.sha }}" 31 | 32 | - name: Set up Python ${{ matrix.python-version }} 33 | uses: actions/setup-python@v2 34 | with: 35 | python-version: ${{ matrix.python-version }} 36 | 37 | - name: Install dependencies 38 | run: | 39 | python -m pip install --upgrade pip 40 | pip install --editable .[test] 41 | 42 | - name: pytest 43 | run: python -m pytest --disable-warnings --cov=fowl -s -v src/fowl/test/ 44 | 45 | - name: Coverage graph 46 | shell: bash 47 | run: cuv graph 48 | continue-on-error: true 49 | 50 | - name: Coverage report 51 | shell: bash 52 | run: | 53 | git diff origin/main..HEAD > p 54 | cuv report p 55 | continue-on-error: true 56 | 57 | - name: Convert to LCOV 58 | shell: bash 59 | run: | 60 | coverage lcov 61 | 62 | - name: Coveralls.io 63 | uses: actions/setup-python@v2 64 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | docs/_build 2 | venv 3 | dist 4 | .coverage 5 | relay.sqlite 6 | PRIVATE* 7 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Set the OS, Python version and other tools you might need 9 | build: 10 | os: ubuntu-22.04 11 | tools: 12 | python: "3.12" 13 | # You can also specify other tool versions: 14 | # nodejs: "19" 15 | # rust: "1.64" 16 | # golang: "1.19" 17 | 18 | # Build documentation in the "docs/" directory with Sphinx 19 | sphinx: 20 | configuration: docs/conf.py 21 | 22 | # Optionally build your docs in additional formats such as PDF and ePub 23 | # formats: 24 | # - pdf 25 | # - epub 26 | 27 | # Optional but recommended, declare the Python requirements required 28 | # to build your documentation 29 | # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 30 | python: 31 | install: 32 | - requirements: docs/requirements.txt 33 | -------------------------------------------------------------------------------- /DEVELOPERS.rst: -------------------------------------------------------------------------------- 1 | 2 | Developing FOW 3 | ============== 4 | 5 | Follow the installation instructions in README.rst using the editable variation:: 6 | 7 | ./venv/bin/pip install --editable . 8 | 9 | Additionally, install the development dependencies:: 10 | 11 | ./venv/bin/pip install --editable .[test] 12 | 13 | 14 | Running the Tests 15 | ----------------- 16 | 17 | The integration tests use `pytest <>`_ and exercise the ``fow`` command-line program directly. 18 | 19 | A pytest Fixture instantiates a local Magic Wormhole "Mailbox Server" running on port 4000. 20 | 21 | To run the suite:: 22 | 23 | ./venv/bin/python -m pytest -s -v integration/ 24 | 25 | To collect coverage:: 26 | 27 | ./venv/bin/python -m pytest --cov=fow -s -v integration/ 28 | 29 | 30 | Future Plans 31 | ------------ 32 | 33 | If you are interested in advancing new features or existing issues with ``fowl`` and related magic-wormhole things, **please get in touch**: 34 | 35 | * Meejah is very often in ``#python`` on the Libera IRC network. 36 | * File an issue on GitHub 37 | 38 | 39 | Specific Future Plans 40 | ~~~~~~~~~~~~~~~~~~~~~ 41 | 42 | Things that we definitely want to do (naming is hard, so those subject to change). 43 | 44 | ``fowld`` is the lowest-level tool. It continues to speak a protocol on stdin/stdout and has minimal CLI options. 45 | 46 | ``fowl`` is the main, human-friendly CLI. It could have a ``--daemon`` option to speak to the above -- otherwise, it runs a ``fowl-daemon`` subprocess itself. Care must be taken, then, that the options make sense as either "one-shot" or not. For example, a ``--listen 80`` option to open a local listener (that forward to remote port 80) is fine -- if there's a ``--daemon`` option then it adds that port to the running daemon; otherwise, it starts one and immediately adds that port. Of course, some commands may simply not make sense for the daemon (or not-daemon) cases, but I think the overlaps will be considerable -- so that's a reason to reject splitting this command. 47 | 48 | 49 | 50 | Concrete Use Cases 51 | ~~~~~~~~~~~~~~~~~~ 52 | 53 | tty-share: ability to share tty-share instances over wormhole. One host, one or many clients. 54 | 55 | ssh: use fowl to interconnect my devices which do not have a public IP and some of them are constantly on the move 56 | 57 | wizard-gardens: See https://meejah.ca/blog/wizard-gardens-vision .. ability to run arbitary glue/plugin code to set up and run various "self-hostable" network applications (a general case of the two above use-cases). 58 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2023 meejah 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: pin release 2 | 3 | lint: 4 | ruff check src/fowl 5 | 6 | pin: 7 | pip-compile --upgrade --allow-unsafe --generate-hashes --resolver=backtracking --output-file requirements-pinned.txt 8 | git add -u 9 | git commit -m "upgrade pins" 10 | 11 | utest: 12 | python -m pytest --cov= --cov-report= -sv -x src/fowl/test 13 | cuv graph 14 | 15 | test: 16 | coverage erase 17 | coverage run --source src/fowl --parallel -m pytest -x --disable-warnings -sv src/fowl 18 | coverage run --source src/fowl --parallel -m pytest -x -v integration/ 19 | coverage combine --append 20 | cuv graph 21 | 22 | #release: pin 23 | release: 24 | python update-version.py 25 | hatch version `git tag --sort -v:refname | head -1` 26 | git add -u 27 | git commit -m "update version" 28 | hatchling build 29 | twine check dist/fowl-`git describe --abbrev=0`-py3-none-any.whl 30 | twine check dist/fowl-`git describe --abbrev=0`.tar.gz 31 | gpg --pinentry=loopback -u meejah@meejah.ca --armor --detach-sign dist/fowl-`git describe --abbrev=0`-py3-none-any.whl 32 | gpg --pinentry=loopback -u meejah@meejah.ca --armor --detach-sign dist/fowl-`git describe --abbrev=0`.tar.gz 33 | 34 | undo-release: 35 | -ls dist/fowl-`git describe --abbrev=0`* 36 | -rm dist/fowl-`git describe --abbrev=0`* 37 | git tag -d `git describe --abbrev=0` 38 | 39 | release-upload: 40 | @ls dist/fowl-`git describe --abbrev=0`* 41 | twine upload --username __token__ --password `cat PRIVATE-release-token` dist/fowl-`git describe --abbrev=0`* 42 | git push github `git describe --abbrev=0` 43 | 44 | wg.png: wizard-garden-app-interaction.seq 45 | # pip install seqdiag 'pillow<10' 46 | seqdiag -T png --no-transparency -o wg.png wizard-garden-app-interaction.seq 47 | -------------------------------------------------------------------------------- /NEWS.rst: -------------------------------------------------------------------------------- 1 | 2 | Fowl Releases 3 | ============= 4 | 5 | Stability: this is still a young project, all APIs should be considered unstable. 6 | That said, ``fowld`` input and output is intended to be stable and compatible. 7 | 8 | Integration with other programs should use ``fowld`` exclusively. 9 | 10 | 11 | Unreleased 12 | ---------- 13 | 14 | * (Put new changelog items here) 15 | * Cool new ASCII-art / terminal visualization of activity (via "rich") 16 | * Allow for non-local addresses: for both listening interfaces and 17 | connect endpoints, non-local addresses may be specified in a manner 18 | similar to "ssh -L" or "ssh -R" arguments. See #37: 19 | https://github.com/meejah/fowl/issues/37 20 | * Fix up some fallout from refactoring 21 | * Enable "remote" command in --interactive 22 | * Proper error-message rendering 23 | * Allow whitelisting only specific connect/listen endpoints. 24 | * Nice error if user gives zero options 25 | 26 | 27 | 24.3.1: March 1, 2024 28 | --------------------- 29 | 30 | * Upgrade dependencies (msgpack, twisted) 31 | 32 | 33 | 24.3.0: March 1, 2024 34 | --------------------- 35 | 36 | * Simplify ``fowl`` to have no sub-commands 37 | * One side runs ``fowl``, the other one runs ``fowl 1-foo-bar`` 38 | * More complete and accurate documentation 39 | 40 | 41 | 24.2.0: February 27, 2024 42 | ------------------------- 43 | 44 | * Extensive refactoring 45 | * ``fowld`` for machines 46 | * ``fowl`` (with ``tui``, ``accept``, ``invite`` subcommands) for humans 47 | * Lots more unit- and integration- tests written 48 | 49 | 50 | 23.10.2: October 18, 2023 51 | ------------------------- 52 | 53 | * Initial release, for gathering feedback 54 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | docs/README.rst -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/README.rst: -------------------------------------------------------------------------------- 1 | Forward over Wormhole, Locally (fowl) 2 | ===================================== 3 | 4 | .. image:: logo.svg 5 | :width: 42% 6 | :align: right 7 | :alt: Fowl Logo: a chicken head with two blue ethernet cables 8 | 9 | Get TCP streams from one computer to another, safely. 10 | 11 | (The base protocol below `Magic Wormhole `_ provides a powerful account-less, peer-to-peer networking solution -- ``fowl`` helps you use this power immediately with existing programs) 12 | 13 | - Code: https://github.com/meejah/fowl 14 | - Documentation: https://fowl.readthedocs.io/en/latest/ 15 | 16 | 17 | 🤔 Motivation 18 | ------------- 19 | 20 | We sometimes pair-program but don't like the idea of sending keystrokes over a third-party server. 21 | That could be solved by self-hosting, but we also like avoiding the extra work of "set up a server on a public IP address". 22 | 23 | For more context, see my blog posts: `Forwarding Streams over Magic Wormhole `_ and `Wizard Gardens vision `_. 24 | 25 | To generalize this a little: there are many FOSS client/server programs that *can* be self-hosted -- ``fowl`` lets us use these sorts of programs in a peer-to-peer fashion, behind NATs. 26 | This means only depending on one general-purpose, public-IP-having server (the Magic Wormhole "mailbox server" used to set up connections) instead of "one per application" (or more). 27 | 28 | .. image:: fowl-interaction-screenshot-1000.png 29 | :height: 474px 30 | :width: 1000px 31 | :alt: screenshot of four terminals demonstrating fowl with two peers, one "nc" and one "telnet": encrypted chat with telnet over wormhole 32 | 33 | 34 | 🦃 What? 35 | -------- 36 | 37 | The command-line tool ``fowl`` allows you use any client/server program over `Magic Wormhole `_. 38 | Magic Wormhole provides a *persistent*, strongly-encrypted session (end-to-end) with no need for pre-shared keys. 39 | 40 | Conceptually, this is somewhat similar to combining ``ssh -R`` and ``ssh -L``. 41 | ``fowl`` may be used to set up complex workflows directly between participants, integrating services that would "traditionally" demand a server on a public IP address. 42 | 43 | Key features: 44 | 45 | * no need to pre-exchange keys 46 | * simple, one-time-use codes that are easy to transcribe 47 | * secure (full-strength keys via SPAKE2) 48 | * end-to-end encryption (and no possibility for unencrypted application data) 49 | * integrate with any tools that can listen on or connect to localhost 50 | 51 | This allows an author to write a "glue" program in *any language* that ties together unchanged networked progams. 52 | The communcation channel is: 53 | 54 | * set up *without pre-shared secrets*; 55 | * *fully encrypted*; 56 | * and *survives IP address changes or outages*. 57 | 58 | All this with *no action required at the application level*, it is just a normal localhost TCP (or UNIX) streaming socket. 59 | 60 | 61 | ✍ Motivational Example 62 | ---------------------- 63 | 64 | When pair-programming using `tty-share `_ one handy option is to use the default, public server. 65 | However, *I don't like the idea of sending keystrokes over a third-party server* that I don't run. 66 | (Please note: I have **no** reason to believe this nice person is doing anything nefarious!) 67 | 68 | I could fire up such a server myself and use it with my friends... 69 | 70 | ...but with ``fowl``, one side can run a localhost ``tty-share`` server and the other side can run a ``tty-share`` client that connects to a ``localhost`` endpoint -- data flows over the wormhole connection (only). 71 | 72 | **Key advantage**: *no need to expose keystrokes to a third-party server*. 73 | 74 | **Additional advantage**: *no need to set up a server on a public IP address*. 75 | 76 | 77 | 🐃 Why is This Particular Yak Being Shorn? 78 | ------------------------------------------ 79 | 80 | I wanted to write a pair-programming application in Haskell, but didn't want to implement Dilation in the Magic Wormhole Haskell library (maybe one day!) 81 | 82 | It also occurred to me that other people might like to experiment with Magic Wormhole (and advanced features like Dilation) in languages that lack a Magic Wormhole implementation -- that is, most of them! 83 | 84 | So, the first step in "write a Haskell pair-programming utility" became "write and release a Python program" :) 85 | 86 | (p.s. the next-higher level Yak is now online at `sr.ht `_ but not "released") 87 | 88 | 89 | ⌨ How Does It Work? 90 | ------------------- 91 | 92 | ``fowl`` uses the "`Dilation `_" feature of the `Magic Wormhole `_ protocol. 93 | 94 | This means that a Magic Wormhole Mailbox server is used to perform a SPAKE2 exchange via a short (but one-time only) pairing code. 95 | For details on the security arguments, please refer to `the Magic Wormhole documentation `_. 96 | After this, an E2E-encrypted direct P2P connection (or, in some cases, via a "transit relay" service) is established between the two computers; 97 | that is, between the computer that created the wormhole code, and the one that consumed it. 98 | 99 | The key encrypting messages on this connection is only known to the two computers; the Mailbox server cannot see any message contents. 100 | (It, like any attacker, could try a single guess at the wormhole code). See the `Magic Wormhole documentation `_ for more details on this. 101 | 102 | The "Dilation" feature further extends the above protocol to provide subchannels and "durability" -- this means the overall connection survives network changes, disconnections, etc. 103 | You can change WiFi networks or put one computer to sleep yet remain connected. 104 | 105 | What ``fowl`` adds is a way to set up any number of localhost listeners on either end, forwarding data over subchannels. 106 | The always-present "control" subchannel is used to co-ordinate opening and closing such listeners. 107 | 108 | With some higher-level co-ordination, ``fowl`` may be used to set up complex workflows between participants, integrating services that would "traditionally" demand a server on a public IP address. 109 | 110 | Another way to view this: streaming network services can integrate the Magic Wormhole protocol without having to find, link, and use a magic-wormhole library (along with the implied code-changes) -- all integration is via local streams. 111 | (There *are* implementations in a few languages so you could take that route if you prefer). 112 | 113 | 114 | 👤 Who Should Use This? 115 | ----------------------- 116 | 117 | We handle and expect two main use-cases of this program: integrators and end-users. 118 | 119 | Human CLI users can use ``fowl`` itself to set up and use connections, for any purpose. 120 | 121 | For developers doing integration, ``fowld`` provides a simple stdin/out protocol for any runtime to use. 122 | That is, some "glue" code running ``fowld`` as a sub-process. 123 | This co-ordination program will also handle running necessary client-type or server-type networking applications that accomplish some goal useful to users. For example, "pair-programming" (for my case). 124 | 125 | Some other ideas to get you started: 126 | 127 | - "private" / invite-only streaming (one side runs video source, invited sides see it) 128 | - on-demand tech support or server access (e.g. set up limited-time SSH, VNC, etc) 129 | - ... 130 | 131 | 132 | 💼 Installation and Basic Usage 133 | ------------------------------- 134 | 135 | ``fowl`` and ``fowld`` are Python programs using the `Twisted `_ asynchronous networking library. 136 | 137 | You may install them with ``pip``:: 138 | 139 | pip install fowl 140 | 141 | Once this is done, ``fowl`` and ``fowld`` will appear on your ``PATH``. 142 | Run either for instructions on use. 143 | 144 | In accordance with best practices, we recommend using a ``virtualenv`` to install all Python programs. 145 | **Never use ``sudo pip``**. 146 | To create a virtualenv in your checkout of ``fowl``, for example: 147 | 148 | .. code-block:: shell 149 | 150 | python -m venv venv 151 | ./venv/bin/pip install --upgrade pip 152 | ./venv/bin/pip install fowl 153 | # or: ./venv/bin/pip install --editable . 154 | ./venv/bin/fowl 155 | 156 | .. _hello-world-chat: 157 | 158 | 💬 Hello World: Chat! 159 | --------------------- 160 | 161 | The actual "hello world" of networked applications these days is chat, amirite? 😉 162 | 163 | We will use two venerable network utilities (``nc`` and ``telnet``) to implement a **simple, secure, and e2e-encrypted chat**. 164 | 165 | Yes, that's correct: we will make secure chat over ``telnet``. 166 | The first insight here is that we can make ``nc`` listen on a localhost-only port, and we can make ``telnet`` connect to a localhost TCP port. 167 | 168 | At first we can prove the concept locally, from one terminal to another. 169 | Open two terminals. 170 | In the first, run: ``nc -l localhost 8888`` 171 | This tells ``nc`` (aka "net cat") to listen on the localhost TCP port "8888" (it will echo anything that comes in, and send anything you type). 172 | 173 | In the second terminal: ``telnet localhost 8888`` 174 | This instructs telnet to connect to localhost TCP port 8888 -- that is, the very netcat instance running in the first terminal. 175 | Type "hello world" into either of the terminals, and you should see it appear on the other side. 176 | 177 | **Goal achieved!**, partially. 178 | We have "chat" over ``nc`` and ``telnet``. 179 | It's not pretty, but it works fine. 180 | 181 | However, we want to talk to other machines. 182 | This means we need: 183 | 184 | * encryption; 185 | * and a way to arrange network connectivity 186 | 187 | **These additional features are exactly what** ``fowl`` **gives us.** 188 | 189 | So, we still run the exact same ``nc`` and ``telnet`` commands, but first do some ``fowl`` magic on each machine. 190 | 191 | On the *first* machine, open a terminal and start ``nc`` on port 8888 via ``nc -l localhost 8888``. We'll then need to add in something that *listens* on port 8888 and sends it through the wormhole. 192 | This thing is: ``fowl --allow-connect 8888 ``. If you don't specify a ````, ``fowl`` will generate one for you, say ``1-foo-bar`` If you want to generate your own codes, you can specify it directly like so: ``fowl --allow-connect 8888 1-foo-bar``. 193 | 194 | On the *second* machine we'll need to add in something that connects our wormhole to our own 8888 port. 195 | This thing is: ``fowl --local 8888 ``, in our case ``fowl --local 8888 1-boo-bar`` 196 | 197 | What happens under the hood is that the two ``fowl`` programs establish a secure connection, via the public Mailbox Server. 198 | They then use this connection to maintain a persistent (possibly changing) TCP connection between each other (worst case, using the public Transit Relay) to send end-to-end encrypted messages. 199 | 200 | ``fowl`` uses this connection to communicate via a simple protocol that can establish listeners on either end or ask for fresh connections. 201 | These result in "subchannels" (in the Magic Wormhole Dilation protocol) that can send bytes back or forth. 202 | 203 | Any bytes received at either end of the connection are simply forwarded over the subchannel. 204 | 205 | Full example, computer one: 206 | 207 | .. code-block:: shell 208 | 209 | $ nc -l localhost 8888 210 | $ fowl --allow-connect 8888 211 | Invite code: 1-foo-bar 212 | 213 | Computer two: 214 | 215 | .. code-block:: shell 216 | 217 | $ fowl --local 8888 1-foo-bar 218 | $ telnet localhost 8888 219 | 220 | **Now we have encrypted chat**. 221 | 222 | These two programs can run **anywhere on the Internet**. 223 | Like TCP promises, all bytes are delivered in-order. 224 | In addition, they are **encrypted**. 225 | Also the stream will **survive changing networks** (disconnects, new IP addresses, etc); that is, the actual inter-computer TCP connection is re-stablished, but to the applications (``nc``, ``telnet``) it looks uninterupted. 226 | 227 | .. note:: 228 | 229 | The two public servers mentioned (the Mailbox Server and the 230 | Transit Relay) will learn the IP addresses of who is 231 | communicating. 232 | 233 | Tor is supported for users who do not wish to reveal their network 234 | location. **Neither server can see any plaintext** (like any 235 | other attacker, the Mailbox Server could try a single but 236 | destructive and noticable guess at the code for any mailbox). 237 | 238 | 239 | 📦 Other Platforms 240 | ------------------ 241 | 242 | We welcome contributions from people experienced with packaging for other installation methods; please get in touch! 243 | 244 | 245 | 🚚 Stability and Releases 246 | ------------------------- 247 | 248 | This is an early release of, essentially, a proof-of-concept. 249 | While we intend to make it a stable base to put co-ordination software on top, it is not yet there. 250 | APIs may change, options may change. 251 | If you are developing on top of ``fowl``, please get in touch so we know what you need 😊 252 | 253 | All releases are on PyPI with versioning following a `CalVer `_ variant: ``year.month.number``, like ``23.4.0`` (for the first release in April, 2023). 254 | 255 | See ``NEWS.rst`` for specific release information. 256 | 257 | 258 | 🧙 Contributors 259 | --------------- 260 | 261 | - `meejah `_: main development 262 | - `shapr `_: much feedback, pairing and feature development 263 | - `balejk `_: early feedback, proof-reading, review and testing 264 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # For the full list of built-in configuration values, see the documentation: 4 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 5 | 6 | # -- Project information ----------------------------------------------------- 7 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 8 | 9 | project = 'fowl' 10 | copyright = '2023-2024, meejah' 11 | author = 'meejah' 12 | 13 | # -- General configuration --------------------------------------------------- 14 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 15 | 16 | extensions = [] 17 | 18 | templates_path = ['_templates'] 19 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 20 | 21 | 22 | 23 | # -- Options for HTML output ------------------------------------------------- 24 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 25 | 26 | html_theme = 'furo' 27 | html_logo = 'logo.svg' 28 | html_static_path = ['.'] 29 | html_theme_options = { 30 | "top_of_page_buttons": ["view", "edit"], 31 | "source_repository": "https://github.com/meejah/fowl/", 32 | "source_branch": "main", 33 | "source_directory": "docs/", 34 | "navigation_with_keys": True, 35 | "footer_icons": [ 36 | { 37 | "name": "GitHub", 38 | "url": "https://github.com/meejah/fowl", 39 | "html": """ 40 | 41 | 42 | 43 | """, 44 | "class": "", 45 | }, 46 | { 47 | "name": "meejah.ca", 48 | "url": "https://meejah.ca/blog/wizard-gardens-vision", 49 | "html": """""", 50 | }, 51 | ], 52 | } 53 | -------------------------------------------------------------------------------- /docs/fowl-forward-light.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meejah/fowl/7cdc2b51bb541d7700f435c3497ab13a6f15646d/docs/fowl-forward-light.png -------------------------------------------------------------------------------- /docs/fowl-logo-0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meejah/fowl/7cdc2b51bb541d7700f435c3497ab13a6f15646d/docs/fowl-logo-0.png -------------------------------------------------------------------------------- /docs/frontend-protocol.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _frontend-protocol: 3 | 4 | The ``fowld`` Frontend Protocol 5 | ================================ 6 | 7 | **Target Audience**: users of ``fowld`` (integrators, GUI authors, experimenters) 8 | 9 | The ``fowld`` program speaks a line-based protocol on stdin and stdout. 10 | 11 | Our intent is to allow any language to use and experiment with Dilation via local streaming interfaces; if a feature for your use-case is missing please `open a new Issue `_. 12 | 13 | Every message is a single line, terminated with a single newline (``\n``) character. 14 | Every message is a complete and valid JSON message, encoded in UTF8. 15 | Every message is a ``dict`` (aka "JSON object"). 16 | 17 | Thus, controller programs can deserialize each line as a ``dict`` (or mapping, or your language's equivalent). 18 | 19 | Since ``fowl`` itself uses ``fowld`` under the hood, this project has examples of parsing and producing these messages. 20 | 21 | We now go over the keys that particular messages have. 22 | 23 | 24 | The ``"kind"`` Key 25 | -------------------- 26 | 27 | Every single message MUST have a ``"kind"`` key. 28 | This tells the parser what sort of message it is (and hence what other keys may be present). 29 | 30 | It is a protocol error to emit a message without a ``"kind"`` key. 31 | Hereafter, we refer to all messages by their "kind". 32 | 33 | An "input" message is one that ``fowld`` accepts on stdin. 34 | 35 | An "output" message is one that ``fowld`` may produce on stdout. 36 | 37 | To follow along in the code, see the ``fowl.messages`` package. 38 | Types derived from ``FowlOutputMessage`` are produced on stdout and ``FowlCommandMessage`` derived types are accepted on stdin. 39 | 40 | 41 | Input: ``kind: allocate-code`` 42 | ------------------------------ 43 | 44 | This message tells ``fowld`` to allocate a fresh code. 45 | These codes look like ``1-foo-bar`` (you can control how many words). 46 | Note that we interact with the server to reserve the "nameplate" (the short number) but the words are chosen locally (and randomly). 47 | 48 | Keys allowed: 49 | 50 | - ``code-length`` (optional): how many words to use 51 | 52 | After issuing this command, at some point in the future a ``kind: code-allocated`` message will be emitted. 53 | 54 | 55 | Input: ``kind: set-code`` 56 | ------------------------- 57 | 58 | Tell ``fowld`` a specific code to use. 59 | This likely came from a corresponding use of ``kind: allocate-code`` (in another instance of ``fowld`` on another computer). 60 | 61 | - ``code`` (required): the secret code to use 62 | 63 | 64 | Input: ``kind: local`` 65 | ---------------------- 66 | 67 | This message asks ``fowld`` to start listening on a local port -- any subsequent connections on this local port cause the *remote* side to open a corresponding connection (possibly on a different port or protocol). 68 | 69 | Keys allowed in this message: 70 | 71 | - ``listen`` (required): a Twisted server-style endpoint string specifying where to listen locally 72 | - ``connect`` (required): a Twisted client-style endpoint string specifying what connection to open on the other end 73 | 74 | For example: 75 | 76 | .. code-block:: json 77 | 78 | { 79 | "kind": "local", 80 | "listen": "tcp:8000:interface=localhost", 81 | "connect": "tcp:localhost:80" 82 | } 83 | 84 | In this example, we will open a listener on our machine on TCP port ``8000`` and interface ``localhost``. 85 | Whenever a new connection is opened on our machine, we will ask the other side to connect to port ``80`` on *their* notion of ``localhost``. 86 | 87 | Remember that these can be anything that Twisted understands, including from installed plugins. 88 | **Be careful**: a malicious "other end" could cause all sorts of shenanigans. 89 | 90 | You are only limited to streaming endpoints (i.e. no UDP) but they do not have to be TCP. 91 | For example, one side could use ``unix:/tmp/socket`` to open a listener (or connection) on Unix-domain socket. 92 | With `txtorcon `_ one could have ``onion:...`` Tor endpoints on either end. 93 | 94 | Once the listener is established, we'll issue a ``kind: listening`` output. 95 | 96 | 97 | Input: ``kind: remote`` 98 | ----------------------- 99 | 100 | This will cause ``fowld`` to request a listener on the *other* side. 101 | There is symmetry here: the same thing could be accomplished by that other side instead issuing a ``kind: local`` request. 102 | 103 | Keys allowed in this message: 104 | 105 | - ``listen`` (required): a Twisted server-style endpoint string specifying where to listen (on the other end). 106 | - ``connect`` (required): a Twisted client-style endpoint string specifying what to connect to (on this side) for each connection that happens on the other side. 107 | 108 | To directly mirror the example from the ``local`` command: 109 | 110 | .. code-block:: json 111 | 112 | { 113 | "kind": "remote", 114 | "listen": "tcp:8000:interface=localhost", 115 | "connect": "tcp:localhost:80" 116 | } 117 | 118 | This will be a mirror-image of the other example. 119 | That is, we'll cause the far end to start listening on its TCP port ``8000`` on interface ``localhost``. 120 | Any connection to that will open a near-side connection to port 80 via TCP. 121 | 122 | The far-side ``fowld`` will issue a ``kind: listening`` message (on its side) when it has started listening. 123 | 124 | 125 | Input: ``kind: grant-permission`` 126 | --------------------------------- 127 | 128 | Each peer needs to have the ability to control what sorts of listeners are opened, and what sort of client-style connections are opened from its network interfaces. 129 | 130 | The wire protocol spoken between Peers has the opportunity to approve or deny every listener, and every connection. 131 | These are known as the "permissions" hooks in the state-machine. 132 | 133 | When `fowld` starts, with no other option, no listeners and no connections will be allowed. 134 | 135 | Messages can be sent to expand what is allowed. 136 | Only "``localhost``" (or ``::1``) interfaces (or destinations) are allowed. 137 | 138 | There are two kinds of policy: "listen" policy and "connect" policy. 139 | These apply to the two kinds of things we may care about: a new listener (governed by the "listen" policy) or a new forwarded connection that needs to open a new client-stype connection (governed by the "connect" policy). 140 | 141 | .. code-block:: json 142 | 143 | { 144 | "kind": "grant-permission", 145 | "listen": [8080], 146 | "connect": [443, 4321] 147 | } 148 | 149 | This will allow a listener on port 8080 (whether initiated remotely or locally), and allow connections to ``localhost:443`` and ``localhost:4321`` for any incoming forwarded connections. 150 | 151 | This is a simple, easy-to-use API but does not reveal all that is possible technically; if the above doesn't fit your use-case, please get in touch by `creating a new Issue <>_`. 152 | 153 | 154 | Input: ``kind: danger-disable-permission-check`` 155 | ------------------------------------------------ 156 | 157 | To facilitate experimentation or other use-cases not available via any other permission API, checking can be turned off entirely. 158 | 159 | .. DANGER:: 160 | 161 | Please understand the implications before enabling this, especially if you do not control both peer computers. 162 | This allows the OTHER peer to open any listener or any connection they like on your machine -- very useful, but easily abused if either side is malicious in any way. 163 | 164 | If you understand that you want this anyway for your side of the connection, send this message 165 | 166 | .. code-block:: json 167 | 168 | { 169 | "kind": "danger-disable-permission-check", 170 | } 171 | 172 | 173 | Output: ``kind: listening`` 174 | --------------------------- 175 | 176 | This message is issued by ``fowld`` when it has opened a listening socket on that side. 177 | 178 | So, if a ``kind: local`` had initiated the listening, this message would appear on that same side. 179 | If instead it was a ``kind: remote`` then it would appear on the far side. 180 | 181 | An example message: 182 | 183 | .. code-block:: json 184 | 185 | { 186 | "kind": "listening", 187 | "listen": "tcp:8080:interface=localhost", 188 | "connect": "tcp:80" 189 | } 190 | 191 | Guidance for UX: the user should be made aware their machine is listening on a particular port / interface. 192 | 193 | 194 | Output: ``kind: error`` 195 | ----------------------- 196 | 197 | Some sort of error has happened. 198 | 199 | This message MUST have a ``message`` key containing a free-form error message. 200 | 201 | An example message: 202 | 203 | .. code-block:: json 204 | 205 | { 206 | "kind": "error", 207 | "message": "Unknown control command: foo" 208 | } 209 | 210 | Guidance for UX: most errors are probably interesting to the user. 211 | 212 | 213 | Output: ``kind: welcome`` 214 | ------------------------ 215 | 216 | This message is emitted to both sides once per session when we connect to the Mailbox Server. 217 | 218 | - ``"welcome"``: a ``dict`` containing whatever the Mailbox server sent in its Welcome message. 219 | 220 | Guidance for UX: the user should be informed that progress has been made (e.g. the Mailbox Server is available). 221 | 222 | 223 | Output: ``kind: code-allocated`` 224 | -------------------------------- 225 | 226 | This is emitted once ``fowld`` has a secret code. 227 | We could have been given one with ``kind: set-code`` or created a new one with ``kind: allocate-code``. 228 | In either case, this message is emitted. 229 | 230 | - ``"code"``: the secret code 231 | 232 | 233 | Output: ``kind: peer-connected`` 234 | -------------------------------- 235 | 236 | The ``fowld`` process has successfully communicated with the other peer. 237 | 238 | - ``"verifier"``: a string containing 32 hex-encoded bytes which are a hash of the session key 239 | - ``"versions"``: an object containing application-specific versioning information 240 | 241 | Guidance for UX: advanced users may wish to compare the verifiers for extra security (they should match; if they don't, it may be a "Machine in the Middle" attack). 242 | 243 | Guidance for integration: the "versions" metadata is intended to allow your application to determine information about the peer. 244 | This could be use for capability discovery, protocol selection, or anything else. 245 | 246 | 247 | Output: ``kind: bytes-in`` 248 | -------------------------- 249 | 250 | The ``fowld`` process received some forwarded bytes successfully. 251 | 252 | Keys present: 253 | 254 | - ``id`` (required): the sub-connection id, a unique number 255 | - ``bytes`` (required): how many bytes are forwarded recently 256 | 257 | Guidance for UX: the user may be curious to know if a connection is alive, what its throughput is, etc. 258 | 259 | 260 | Output: ``kind: bytes-out`` 261 | --------------------------- 262 | 263 | The ``fowld`` process forwarded some bytes to the other peer successfully. 264 | 265 | Keys present: 266 | 267 | - ``id`` (required): the sub-connection id, a unique number 268 | - ``bytes`` (required): how many bytes are forwarded recently 269 | 270 | Guidance for UX: the user may be curious to know if a connection is alive, what its throughput is, etc. 271 | 272 | 273 | Output: ``kind: local-connection`` 274 | ---------------------------------- 275 | 276 | We have received a connection on one of our local listeners. 277 | 278 | Keys present: 279 | 280 | - ``id`` (required): the sub-connection id, a unique number 281 | 282 | Guidance for UX: the user should be informed that something is interacting with our listener. 283 | 284 | 285 | Output: ``kind: incoming-connection`` 286 | ------------------------------------- 287 | 288 | The other side has asked us to make a local connection. 289 | 290 | Keys present: 291 | 292 | - ``id`` (required): the sub-connection id, a unique number 293 | - ``endpoint`` (required): the Twisted client-style endpoint we will attempt a connection to 294 | 295 | Guidance for UX: the user should be informed that something is interacting with our listener. 296 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | 2 | fowl: Forward Over Wormhole, Locally 3 | ======================================= 4 | 5 | ``fowl`` is a tool that utilizes `Magic Wormhole `_ and its Dilation feature to forward arbitrary streams (TCP, Unix, stdin, etc) over an easy-to-setup yet secure connection. 6 | **Peers** communicate to each other over an **end-to-end encrypted** connection, and can use client-type or server-type network services from each side. 7 | Allowed ports and permissions are **whitelist** based. 8 | 9 | .. image:: _static/fowl-forward-light.png 10 | :width: 100% 11 | :alt: Fowl forwarding a connection, with traffic visualization and some ascii-art showing connected peers 12 | 13 | There are no logins, no identities and the server can't see content because everything is end-to-end (E2E) encrypted between exactly two peers. 14 | Additionally, the server is often not involved in the "bulk transport" of bytes at all as the protocol prefers P2P connections. 15 | 16 | Conceptually, this is similar to ``ssh -R`` and ``ssh -L``. 17 | 18 | Sound interesting? Read on! 19 | 20 | .. toctree:: 21 | :maxdepth: 2 22 | 23 | README 24 | usage 25 | releases 26 | frontend-protocol 27 | protocol 28 | -------------------------------------------------------------------------------- /docs/logo.svg: -------------------------------------------------------------------------------- 1 | ../logo.svg -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | %SPHINXBUILD% >NUL 2>NUL 14 | if errorlevel 9009 ( 15 | echo. 16 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 17 | echo.installed, then set the SPHINXBUILD environment variable to point 18 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 19 | echo.may add the Sphinx directory to PATH. 20 | echo. 21 | echo.If you don't have Sphinx installed, grab it from 22 | echo.https://www.sphinx-doc.org/ 23 | exit /b 1 24 | ) 25 | 26 | if "%1" == "" goto help 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/protocol.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _dilation-protocol: 3 | 4 | Audience: developers of fowl. 5 | 6 | This describes a low-level protocol used by fowl itself; users will never see this. 7 | 8 | 9 | Dilation Application Protocol 10 | ============================= 11 | 12 | Once a "Dilated Magic Wormhole" has been established, we have: 13 | 14 | - a connection to our peer; 15 | - a "control" subchannel; 16 | - the ability to open more subchannels; 17 | 18 | All subchannels are in-order, reliable and durable message-pipes. 19 | "Durable" and "reliable" here means that the underlying Dilation code does the hard work of re-connecting to the peer -- we don't have to worry about acknowledgements, re-connection, re-trying, etc. 20 | 21 | Thus it can sometimes take more time than expected for bytes to be delivered (for example, if some re-connecting is going on behind the scenes). 22 | 23 | 24 | Overall Philosophy 25 | ------------------ 26 | 27 | The program `fowld` is providing a way for a control program (possibly the `fowl` CLI or TUI, or a third-party GUI) to open and forward streams. 28 | Thus when we say "user" in this document, we mean that control program. 29 | Messaging to human users will be via the control program (i.e. they won't see the JSON messages described here directly, usually). 30 | 31 | We want this to be as easy as possible, but also to mimic the "real" Dilation experience -- for many use-cases, `fowl` may be a "stepping stone" to using a language-specific implementation of Dilation (if / when non-Python ones arrive). 32 | 33 | 34 | General Message Format 35 | ---------------------- 36 | 37 | All structured messages on the control or any other channel use `msgpack`_ for encoding. 38 | Every message starts with 2 bytes indicating its length. 39 | Then follow that many bytes constituting a complete `msgpack`_ message. 40 | 41 | An exception to the above is for subchannels: once they switch to "forwarding mode", raw stream bytes are forwarded (not `msgpack`_ messages). 42 | 43 | Remember that Dilation subchannels are actually _record_ pipes -- they are sending authenticated, encrypted, length-delimited messages. 44 | 45 | 46 | Control Subchannel 47 | ------------------ 48 | 49 | XXX: I think we maybe want "reply-id's" on the requests, and one reply (eventually) per request. 50 | 51 | -> opening a listener can take arbitrary time (e.g. consider an 52 | "onion" listener that launches tor), and can fail for a number of 53 | reasons 54 | -> the sending side would like to know if the listen failed, or succeeded 55 | -> one use-case is tests (currently it looks for a "listening" signal 56 | from the other side, we could add a "rejected" message too, but an 57 | explicit req/reply might better ... even if we have a user visible "rejected 58 | listen" message _as well)) 59 | 60 | We use the control subchannel to send requests to the other peer. 61 | 62 | So, when we're asked to open a listener on the far end, we send a message over the control channel. 63 | All control messages decode to a ``dict`` and will have at least a ``kind`` key. 64 | 65 | ``"kind": "remote-to-local"`` 66 | ````````````````````````````` 67 | 68 | An incoming message of this kind instructs us to open a local listener. 69 | Upon any connection to this listener, we open a "forwarding subchannel" (see next section). 70 | 71 | The message comes with some data:: 72 | 73 | { 74 | "kind": "remote-to-local", 75 | "listen-endpoint": "tcp:1234:interface=localhost", 76 | "connect-endpoint": "tcp:localhost:4444", 77 | "reply-id": 9381 78 | } 79 | 80 | This says that we should open a local listener on ``"listen-endpoint"`` -- which is to say on ``"tcp:1234:interface=localhost"``, which is a Twisted server-style endpoint string (so can be passed to ``serverFromString()``). 81 | 82 | The ``"reply-id"`` must be a unique identifier for this message; it may be "retired" once a reply has been sent over the control channel. 83 | There will be exactly one reply; a positive one if we have successfully listened or a negative on if that failed. 84 | Setting up a listener may take arbitrary time (consider a Tor "Onion service" listener thay may need to launch Tor, etc). 85 | Software on the other end may ask its human to allow/deny connections -- humans may take considerable time to answer. 86 | Failures may be due to policy (e.g. the other peer refuses to listen on an interface or port) or technical errors (port already in use, as but one example). 87 | 88 | The reply-id may be any number between 1 and 2^53 - 1 (to support JavaScript implementations). 89 | We recommend simply starting at 1 and incrementing for each request. 90 | 91 | Replies look like:: 92 | 93 | { 94 | "kind": "listener-response", 95 | "reply-id": 9381, 96 | "listening": True 97 | } 98 | 99 | The `"reply-id"` MUST match a previously-received outstanding request. 100 | Only a single response may ever be made for a particular `"reply-id"`. 101 | The `"listening"` boolean indicates if the request succeeded or not. 102 | 103 | For "negative" responses, a reason may be included:: 104 | 105 | { 106 | "kind": "listener-response", 107 | "reply-id": 9381, 108 | "listening": False, 109 | "reason": "Against local policy" 110 | } 111 | 112 | 113 | Upon every connection to this local port (assuming a listener is established), we will open a "forwarding subchannel" to the other side (see next section); ``"connect-endpoint"`` is used here. 114 | 115 | So in this case, for every local connection on port 1234 a subchannel is opened to the other side, and an initial `msgpack`_ message asking to connect to ``"tcp:localhost:4444"`` is sent. 116 | This string is a Twisted *client*-style endpoint string (so ``clientFromString()`` can parse it). 117 | The other side sends a reply when they've connected (or failed). 118 | After this, the connection switches to simply forward all received bytes back for forth. 119 | 120 | Notice in this example the ports are different! 121 | That's okay, but it will be more common to use the very same port. 122 | 123 | Ports are **especially important for Web applications** which often fail if the ports don't line up (because browsers consider the port part of the Origin). 124 | So if you are forwarding Web (or WebSocket) connections, you'll probably want the same port on both sides. 125 | 126 | Because we use Twisted endpoint strings, many protocols are possible on either side: unix-sockets, tor network connections, or anything that supports the appropriate interfaces. 127 | 128 | .. WARNING:: 129 | 130 | This flexibility can be both good and bad; part of the stdin/out protocol can include a "consent" API allowing controlling applications (ultimately, users) to allow or deny each connection or listener. 131 | If you do this, **we recommend whitelisting** only known-good kinds of strings for most users. 132 | 133 | .. _forwarding-subchannel: 134 | 135 | Forwarding Subchannel 136 | --------------------- 137 | 138 | A forwarding subchannel is opened whenever a new connection to a listener is made. 139 | There is a brief handshake, and then the connection merely forwards bytes as they are received (from either end). 140 | 141 | The handshake consists of the initiating side sending a single length-prefixed `msgpack`_ messsage (the length is an unsigned short, two bytes). 142 | The handshake message decodes to a ``dict`` consisting of:: 143 | 144 | { 145 | "local-destination": "tcp:localhost:4444", 146 | } 147 | 148 | This tells the side where to connect. 149 | If it is "okay" to connect to this endpoint (per policy, or the consent API) that is attempted. 150 | 151 | Once the connection succeeds or fails (or, fails to pass policy) a reply message is sent back. 152 | The reply message is also an unsigned-short-prefixed `msgpack`_ message which is a ``dict``:: 153 | 154 | { 155 | "connected": True, 156 | } 157 | 158 | If this is ``False`` then an error occurred and the subchannel should be closed. 159 | Otherwise the connection switches to forwarding data back and forth. 160 | 161 | XXX: consider adding a "reason" string to the reply? 162 | 163 | No bytes shall be forwarded until the reply is received; once the reply is received only forwarded bytes occur on the subchannel (no more structured messages). 164 | 165 | Note that there may be multiple subchannels open "at once" so an application may asynchronously open and await the completion of an arbitrary number of connections. 166 | 167 | 168 | .. _msgpack: https://msgpack.org 169 | -------------------------------------------------------------------------------- /docs/releases.rst: -------------------------------------------------------------------------------- 1 | Releases 2 | ======== 3 | 4 | This software is still considered experimental. 5 | We will start publishing release notes at a future time. 6 | 7 | Any release my introduce breaking changes to any of the protocols or options. 8 | 9 | next 10 | ---- 11 | 12 | *(Likely to be released as 25.x.y)* 13 | 14 | - First release containing these release notes. 15 | - Use ``app_versions`` to properly indicate our version to the peer. 16 | - debug message output 17 | - ``status`` command in the TUI 18 | - ping/pong 19 | - use ``furo`` theme for docs; much editing 20 | - "policy" and "permissions" APIs 21 | 22 | 23 | 24.3.2 24 | ------ 25 | 26 | Last release before this document existed. 27 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | furo 2 | -------------------------------------------------------------------------------- /docs/usage.rst: -------------------------------------------------------------------------------- 1 | ``fowld`` versus ``fowl`` 2 | ========================= 3 | 4 | This project actually ships two CLI tools: ``fowld`` and ``fowl``. 5 | 6 | One is intended for automated, programmatic use (``fowld``) and the other is intended for human use (``fowl``). 7 | 8 | Most users should use ``fowl``. 9 | 10 | Programs that integrate with (or otherwise want stable, machine-parsable output) should use ``fowld``. 11 | Under the hood, ``fowl`` commands actually use ``fowld`` (via a Python API). 12 | All functionality is available to users of either program. 13 | 14 | If you want very similar operation to ``fowld`` but do not like typing JSON, use ``fowl tui``. 15 | 16 | 17 | High-Level Overview 18 | ------------------- 19 | 20 | What we aim to accomplish here is to easily set up the forwarding of TCP or Unix streams over a secure, identity-less and durable connection. 21 | 22 | These streams may be anything at all -- but the core use-case is aimed at eliminating the need to run public-IP services. 23 | Our canonical "hello world" example is a simple chat system: 24 | - running ``nc`` (aka "netcat") on one side; 25 | - and ``telnet`` on the other (see :ref:`hello-world-chat` for a fully-worked example). 26 | 27 | Although ``nc`` and ``telnet`` provide no security, using them here we get an end-to-end-encrypted chat session. 28 | We also get "durability" (if one side loses conenction or changes to a different network, we will eventually resume uninterrupted). 29 | We do not have to change the ``nc`` or ``telnet`` programs at all (they can already connect to and listen upon ``localhost`` -- that's all we need). 30 | 31 | The general flow of a session is that one side "starts" it (allocating a secret code), and the second side "joins" it (consuming the secret coe). 32 | These codes can be used precisely once: if a bad actor guesses or intercepts the code, your partner will know (because it won't work for them). 33 | 34 | You may also gain additional security by using the "verifier" feature, if desired (this ensures that you're 100% definitely communicating with the intended party). 35 | See the `magic-wormhole documentation `_ for a full security discussion. 36 | 37 | 38 | Philsophy of Commands 39 | --------------------- 40 | 41 | The ``fowl`` program accepts human-typed arguments, asks questions that humans are expected to answer and produces messages for humans to read. 42 | Many options are available via normal command-line arguments. 43 | 44 | Although we'll still avoid gratuitous compatilibity problems, the output SHOULD NOT be considered machine-parsable and may change from release to release. 45 | 46 | By contrast, the commands that ``fowld`` accepts and the messages it outputs MUST all be well-formed JSON lines. 47 | Generally, backwards-compatibility SHOULD be available. 48 | 49 | There should be few (ideally no) command-line options for ``fowld``. 50 | Programs integrating with it should be able to use any version of the software (that is, to upgrade seamlessly). 51 | 52 | .. note:: 53 | 54 | Since this is still in rapid development we don't make any promises 55 | about backwards compatibility *yet*, but will expect in future to 56 | have a protocol version that will increment with any breaking 57 | changes. 58 | 59 | 60 | ``fowl`` Usage 61 | ============== 62 | 63 | ``fowl`` is a friendly, human-centric frontend to start or join a forwarding session. 64 | You may specify streams to forward and rules to accept forwarded streams. 65 | 66 | We are cautious by default, so any incoming stream requests will result in a "y/n" style question on the command-line (unless overridden by options specifically allowing streams). 67 | 68 | Since the Dilation protocol is fairly symmetric, most options are available under ``fowl`` instead of the sub-commands ``fowl accept`` and ``fowl invite`` 69 | 70 | For example, whether you started or joined a session, either side can ask the other side to start forwarding a port (``--remote``) or start one on the near side (``--local``). 71 | Thus, the options for what to allow are required on both sides. 72 | 73 | 74 | Overview of a Session 75 | --------------------- 76 | 77 | Using ``fowl`` involves two computers. 78 | One computer runs ``fowl invite`` and the other computer runs ``fowl accept``. 79 | 80 | After this, a lot of things are "symmetric" in that either side can listen on a port (or cause the peer to listen on a port) and subsequently forward data over resulting connections. 81 | 82 | The "symmetric" parts are described in the next session, following which are things specific to the "accept" or the "invite" side. 83 | 84 | This wording can become a little confusing due to the symmetry. 85 | Basically, either peer can set up a listener. 86 | When doing so, you must take care set up permissions on the *other* peer. 87 | 88 | Thus, if you have a ``--local`` on one peer you should expect a corresponding ``--allow-connect`` on the other peer. 89 | Similarly, if there is a ``--remote`` on one peer, you should expect the other peer to require a corresponding ``--allow-listen`` argument. 90 | 91 | 92 | .. image:: _static/magic-peer-networking.svg 93 | 94 | 95 | So, in the above we have "Peer A" running a Web server (in this case Twisted's) that it wishes to expose to Peer B's "curl" command. 96 | The "Controlling App" on "Peer A" runs the "twisted web" as a subprocess, and also a "fowld" as a subprocess. 97 | Similar on the "Peer B" side: it also runs a "fowld" and, in this case, the "client" application. 98 | 99 | .. NOTE:: 100 | 101 | Although we explain this example using ``fowl`` options, usually a 102 | controlling application like this would use ``fowld``. 103 | 104 | 105 | There are two ways to set up the desired flow in ``fowl``! 106 | 107 | One way is for "Peer A" to direct its "fowl" to do a "remote" listener (e.g. with ``--remote 4321:8080``) which says to listen on "4321" on the far-side peer (i.e. "Peer B") and forward connections to "8080" on the near side (i.e. "Peer A"). 108 | "Peer B" will check its permission (e.g. ``--allow-listen 4321``) before actually listening. 109 | 110 | The other way is for "Peer B" to direct its "fowl" to do a "local" listener (e.g. with ``--local 4321:8080``) which says to listen on "4321" on the near-side peer (i.e. "Peer B") and to forward connections to "8080" on the far side (i.e. "Peer A"). 111 | 112 | These are both pretty equivalent, because they end up with the same situation: Peer A has "server-style" application running on port 8080, and Peer B makes it *look* like it's accessible there. 113 | The listener on Peer B is "fowl". 114 | That is, "twistd web" is listening on 8080 on Peer A and "fowl" is listening on "4321" on Peer B. 115 | 116 | When "curl" runs on Peer B, the fowl on Peer B sees a connection, and opens a "dilation subchannel" to Peer A. 117 | It then sends an initial ``msgpack``-encoded message asking for ``local-destination`` of ``tcp:localhost:8080``. 118 | Peer B checks its policy (e.g. ``--allow-connect 8080``) and replies good or bad. 119 | If good, all data is forwarded across the subchannel. 120 | 121 | Choosing one over the other is up to the "Controlling Application". 122 | In this example, the "Controlling Application" could be a Web preview or collaboration tool where "Peer A" has the Web site files. 123 | "Peer B" can then see the proposed Web site. 124 | 125 | 126 | Common ``fowl`` Options: An Example 127 | ----------------------------------- 128 | 129 | Both subcommands ``accept`` and ``invite`` share a series of options for setting up streaming connections. 130 | 131 | Either side may have a listener on a local port; this listener will accept any incoming connection, create a Wormhole subchannel, and ask the other side to make a particular local connection. 132 | 133 | The normal use-case here is that you're running a daemon on one of the two peers and you wish to have the other peer be able to reach it. 134 | 135 | Let's take SSH as an example: the computer "desktop" is running an SSH daemon on the usual port 22. 136 | One this side we run ``fowl invite``, which produces a code. 137 | 138 | On the computer called "laptop" we run ``fowl accept``, consuming the code. 139 | 140 | So to use SSH over this Wormhole connnection, we want to have a listener appear on the "laptop" (because the "desktop" computer already has a listener: the SSH daemon on port 22). 141 | 142 | We have two choices here: either the "desktop" or the "laptop" side may initiate the listening; if we do it on the "desktop" side we use the ``"remote"`` command and if we do it on the "laptop" side we use the ``"local"`` command. 143 | 144 | The ``"remote"`` and ``"local"`` commands are mirrors of each other and both have a ``"listen"`` and ``"connect"`` value -- what changes is _where_ that value is used. 145 | In a ``"remote"`` command, the ``"listen"`` value is used on the "far" side, whereas in a ``"local"`` command the ``"listen"`` value is used on the near side. 146 | 147 | So back to our example, we want the "laptop" to open a new listener. 148 | 149 | On the "laptop" machine we'd use something like ``--local 22`` to indicate that we'd like to listen on port ``22`` (and forward to the same port on the other side). 150 | Maybe we can't listen on ``22``, though, so we might want to listen on ``1234`` but still forward to ``22`` on the far side; this is expressed with ``--local 1234:22`` 151 | 152 | To flip this around, on the "desktop" machine we could do ``--remote 22`` or ``--remote 1234:22`` to use the same values from above. 153 | 154 | .. NOTE:: 155 | 156 | If you're using ``fowld`` directly, the above correspond to ``{"kind": "remote", "listen": "tcp:1234:interface=localhost", "connect": "tcp:localhost:22}`` from the "desktop" machine or ``{"kind": "local", "listen": "tcp:1234:interface=localhost", "connect": "tcp:localhost:22}`` from the "laptop" machine. 157 | 158 | 159 | Common ``fowl`` Options 160 | ----------------------- 161 | 162 | * ``--local port:[remote-port]``: listen locally on ``port``. On any connection to this port, we will ask the peer to open a connection on its end to ``port`` (instead to ``remote-port`` if specified). 163 | 164 | * ``--remote port:[local-port]``: listen on the remote peer's ``port``. On any connection to this port (on the peer's side), we will ask our local side to open a connection to ``port`` (or instead to ``local-port`` if specified). 165 | 166 | 167 | Starting a Session 168 | ------------------ 169 | 170 | One side has to begin first, and this side runs ``fowl`` (possibly with some options). 171 | This uses the Magic Wormhole protocol to allocate a short, one-time code. 172 | 173 | This code is used by the "other end" to join this forwarding session with ``fowl ``. 174 | Once that side has successfully set up, we will see a message:: 175 | 176 | Peer is connected. 177 | Verifier: b191 e9d1 fd27 be77 f576 c3e7 f30d 1ff3 e9d3 840b 7f8e 1ce2 6730 55f4 d1fc bb4f 178 | 179 | After this, we reach the more "symmetric" state of the session: although under the hood one side is randomly "the Follower" and one side is "the Leader" in the Dilation session, at our level either side can request forwards from the other. 180 | 181 | The "Verifier" is a way to confirm that the session keys match; confirming both sides have the same verifier is optional. 182 | However, confirming them means you can be 100% sure (instead of 99.85% sure or 1 in 65536) nobody has become a MitM. 183 | 184 | See below. 185 | 186 | 187 | Joining a Session 188 | ----------------- 189 | 190 | One side has to be the "second" user to a session and that person runs this command. 191 | ``fowl `` consumes a Wormhole code and must receive it from the human who ran the ``fowl`` command which allocated the code. 192 | 193 | Once the Magic Wormhole protocol has successfully set up a Dilation connection, a message will appear on ``stdout``:: 194 | 195 | Peer is connected. 196 | Verifier: b191 e9d1 fd27 be77 f576 c3e7 f30d 1ff3 e9d3 840b 7f8e 1ce2 6730 55f4 d1fc bb4f 197 | 198 | After this, we reach the more "symmetric" state of the session: although under the hood one side is randomly "the Follower" and one side is "the Leader" in the Dilation session, at our level either side can request forwards from the other. 199 | 200 | Generally ports to forward are specified on the command-line (and "policy" type options to allow or deny these are also expressed as command-line options). 201 | In case no "policy" options were specified, the user will be interactively asked on every stream that the other side proposes to open. 202 | 203 | 204 | ``fowld`` Usage 205 | =============== 206 | 207 | ``fowld`` is a command-line tool intended to be run in a terminal session or as a subprocess by a higher-level co-ordination program (e.g. a GUI, or a WAMP client, or ``fowl``). 208 | 209 | All interactions (besides CLI options) are via a line-based protocol: each line is a complete JSON object. 210 | 211 | Most humans should use ``fowl`` instead. 212 | 213 | See :ref:`frontend-protocol` for details on the stdin / stdout protocol that is spoken by ``fowld``. 214 | 215 | 216 | ``fowl --interactive`` Usage 217 | ============================ 218 | 219 | Mostly aimed at developers or advanced usage, this command essentially directly maps the frontend protocol (see :ref:`frontend-protocol`) to interactive commands. 220 | 221 | At the ``>>>`` prompt, certain commands are accepted. 222 | These map directly to ``"kind"` JSON commands from the above-referenced protocol. 223 | 224 | That is, you _could_ just run ``fowld`` and type in JSON directly -- but this is a little nicer! 225 | 226 | There is also a ``status`` command that shows our current knowledge of listeners and active connections. 227 | For debugging, it can sometimes be useful to use the ``ping`` command. 228 | -------------------------------------------------------------------------------- /fowl-interaction-screenshot-1000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meejah/fowl/7cdc2b51bb541d7700f435c3497ab13a6f15646d/fowl-interaction-screenshot-1000.png -------------------------------------------------------------------------------- /integration/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import pytest_twisted 3 | from util import ( 4 | WormholeMailboxServer, 5 | ) 6 | 7 | 8 | @pytest.fixture(scope='session') 9 | def reactor(): 10 | # this is a fixture in case we might want to try different 11 | # reactors for some reason. 12 | from twisted.internet import reactor as _reactor 13 | return _reactor 14 | 15 | 16 | @pytest_twisted.async_fixture() # XXX #56 in pytest-twisted :( (scope='session') 17 | async def wormhole(reactor, request): 18 | """ 19 | A local Magic Wormhole mailbox server 20 | """ 21 | return await WormholeMailboxServer.create( 22 | reactor, 23 | request, 24 | ) 25 | -------------------------------------------------------------------------------- /integration/test_happy.py: -------------------------------------------------------------------------------- 1 | 2 | import pytest_twisted 3 | from twisted.internet.protocol import Factory, Protocol 4 | from twisted.internet.defer import Deferred 5 | from twisted.internet.endpoints import serverFromString, clientFromString 6 | 7 | from util import fowld 8 | 9 | from fowl.messages import * 10 | 11 | # since the src/fowl/test/test_forward tests exercise the underlying 12 | # "fowld" functionality (minus the entry-point), this tests the 13 | # "user-friendly" frontend. 14 | 15 | 16 | class HappyListener(Protocol): 17 | def __init__(self): 18 | self._waiting = [] 19 | 20 | def when_made(self): 21 | d = Deferred() 22 | self._waiting.append(d) 23 | return d 24 | 25 | def dataReceived(self, data): 26 | print(f"unexpected client data: {data}") 27 | 28 | def connectionMade(self): 29 | self.transport.write(b"some test data" * 1000) 30 | self._waiting, waiting = [], self._waiting 31 | for d in waiting: 32 | d.callback(None) 33 | self.transport.loseConnection() 34 | 35 | 36 | class HappyConnector(Protocol): 37 | """ 38 | A client-type protocol for testing. Collects all data. 39 | """ 40 | 41 | def connectionMade(self): 42 | self._data = b"" 43 | self._waiting_exit = [] 44 | 45 | def dataReceived(self, data): 46 | self._data += data 47 | 48 | def connectionLost(self, reason): 49 | self._waiting_exit, waiting = [], self._waiting_exit 50 | for d in waiting: 51 | d.callback(self._data) 52 | 53 | def when_done(self): 54 | """ 55 | :returns Deferred: fires when the connection closes and delivers 56 | all data so far 57 | """ 58 | d = Deferred() 59 | self._waiting_exit.append(d) 60 | return d 61 | 62 | 63 | # could use hypothesis to try 'a bunch of ports' but fixed ports seem 64 | # easier to reason about to me 65 | @pytest_twisted.ensureDeferred 66 | async def test_happy_remote(reactor, request, wormhole): 67 | """ 68 | A session forwarding a single connection using the 69 | ``kind="remote"`` command. 70 | """ 71 | f0 = await fowld(reactor, request, mailbox=wormhole.url) 72 | msg = await f0.protocol.next_message(Welcome) 73 | f0.protocol.send_message({"kind": "allocate-code"}) 74 | code_msg = await f0.protocol.next_message(CodeAllocated) 75 | 76 | # normally the "code" is shared via human interaction 77 | 78 | f1 = await fowld( 79 | reactor, request, 80 | mailbox=wormhole.url 81 | ) 82 | await f1.protocol.next_message(Welcome) 83 | f1.protocol.send_message({"kind": "set-code", "code": code_msg.code}) 84 | await f1.protocol.next_message(CodeAllocated) 85 | 86 | await f0.protocol.next_message(PeerConnected) 87 | await f1.protocol.next_message(PeerConnected) 88 | 89 | # remote side will fail to listen if we don't authorize permissions 90 | f0.protocol.send_message({ 91 | "kind": "grant-permission", 92 | "listen": [1111], 93 | "connect": [], 94 | }) 95 | f1.protocol.send_message({ 96 | "kind": "grant-permission", 97 | "listen": [], 98 | "connect": [8888], 99 | }) 100 | 101 | # open a listener of some sort 102 | f1.protocol.send_message({ 103 | "kind": "remote", 104 | "listen": "tcp:1111:interface=localhost", 105 | "connect": "tcp:localhost:8888", 106 | }) 107 | 108 | # f1 sent a remote-listen request, so f0 should receive it 109 | msg = await f0.protocol.next_message(Listening) 110 | assert isinstance(msg, Listening) 111 | assert msg.listen == "tcp:1111:interface=localhost" 112 | assert msg.connect == "tcp:localhost:8888" 113 | 114 | ep0 = serverFromString(reactor, "tcp:8888:interface=localhost") 115 | ep1 = clientFromString(reactor, "tcp:localhost:1111") 116 | 117 | # we listen on the "real" server interface 118 | port = await ep0.listen(Factory.forProtocol(HappyListener)) 119 | request.addfinalizer(port.stopListening) 120 | # ...and connect via the "local" proxy/listener (so this 121 | # connection goes over the wormhole) 122 | client = await ep1.connect(Factory.forProtocol(HappyConnector)) 123 | 124 | # extract the data 125 | data0 = await client.when_done() 126 | assert data0 == b"some test data" * 1000 127 | 128 | forwarded = await f1.protocol.next_message(BytesIn) 129 | assert forwarded.bytes == len(b"some test data" * 1000) 130 | 131 | 132 | @pytest_twisted.ensureDeferred 133 | async def test_happy_local(reactor, request, wormhole): 134 | """ 135 | A session forwarding a single connection using the 136 | ``kind="local"`` command. 137 | """ 138 | f0 = await fowld(reactor, request, mailbox=wormhole.url) 139 | f0.protocol.send_message({"kind": "danger-disable-permission-check"}) 140 | f0.protocol.send_message({"kind": "allocate-code"}) 141 | code_msg = await f0.protocol.next_message(CodeAllocated) 142 | 143 | 144 | # normally the "code" is shared via human interaction 145 | 146 | f1 = await fowld(reactor, request, mailbox=wormhole.url) 147 | f1.protocol.send_message({"kind": "danger-disable-permission-check"}) 148 | f1.protocol.send_message({"kind": "set-code", "code": code_msg.code}) 149 | # open a listener of some sort 150 | f1.protocol.send_message({ 151 | "kind": "local", 152 | "listen": "tcp:8888:interface=localhost", 153 | "connect": "tcp:localhost:1111", 154 | }) 155 | 156 | await f0.protocol.next_message(PeerConnected) 157 | await f1.protocol.next_message(PeerConnected) 158 | 159 | # f1 send a remote-listen request, so f0 should receive it 160 | msg = await f1.protocol.next_message(Listening) 161 | assert isinstance(msg, Listening) 162 | assert msg.listen == "tcp:8888:interface=localhost" 163 | assert msg.connect == "tcp:localhost:1111" 164 | 165 | ep0 = serverFromString(reactor, "tcp:1111:interface=localhost") 166 | ep1 = clientFromString(reactor, "tcp:localhost:8888") 167 | 168 | # listen on the "real" server address 169 | port = await ep0.listen(Factory.forProtocol(HappyListener)) 170 | request.addfinalizer(port.stopListening) 171 | # ...and connect via the configured "local listener" (so this goes 172 | # via the wormhole) 173 | client = await ep1.connect(Factory.forProtocol(HappyConnector)) 174 | 175 | data0 = await client.when_done() 176 | assert data0 == b"some test data" * 1000 177 | 178 | b = await f0.protocol.next_message(BytesIn) 179 | assert b.bytes == len(b"some test data" * 1000) 180 | -------------------------------------------------------------------------------- /integration/test_human.py: -------------------------------------------------------------------------------- 1 | import re 2 | import socket 3 | 4 | import pytest_twisted 5 | from twisted.internet.interfaces import ITransport 6 | from twisted.internet.protocol import ProcessProtocol, Factory, Protocol 7 | from twisted.internet.task import deferLater 8 | from twisted.internet.defer import Deferred 9 | from twisted.internet.endpoints import serverFromString, clientFromString 10 | from attrs import define 11 | 12 | from util import run_service 13 | 14 | # since the src/fowl/test/test_forward tests exercise the underlying 15 | # "fowld" functionality (minus the entry-point), this tests the 16 | # "user-friendly" frontend. 17 | 18 | 19 | @define 20 | class _Fowl: 21 | transport: ITransport 22 | protocol: ProcessProtocol 23 | 24 | 25 | class _FowlProtocol(ProcessProtocol): 26 | """ 27 | This speaks to an underlying ``fowl`` sub-process. 28 | """ 29 | 30 | def __init__(self, reactor): 31 | self._reactor = reactor 32 | self.exited = Deferred() 33 | self._data = "" 34 | self._waiting = [] 35 | 36 | def processEnded(self, reason): 37 | self.exited.callback(None) 38 | 39 | async def have_line(self, regex): 40 | d = Deferred() 41 | self._waiting.append((d, regex)) 42 | self._maybe_trigger() 43 | return await d 44 | 45 | def _maybe_trigger(self): 46 | lines = [ 47 | line 48 | for line in self._data.split("\n") 49 | if line.strip() 50 | ] 51 | for i, item in enumerate(self._waiting): 52 | d, regex = item 53 | for line in lines: 54 | m = re.match(regex, line) 55 | if m: 56 | del self._waiting[i] 57 | d.callback(m) 58 | return 59 | 60 | def childDataReceived(self, childFD, data): 61 | self._data += data.decode("utf8") 62 | self._maybe_trigger() 63 | 64 | 65 | async def fowl(reactor, request, subcommand, *extra_args, mailbox=None): 66 | """ 67 | Run `fowl` with a given subcommand 68 | """ 69 | 70 | args = [ 71 | "fowl.cli", 72 | ] 73 | if mailbox is not None: 74 | args.extend([ 75 | "--mailbox", mailbox, 76 | ]) 77 | 78 | args.append(subcommand) 79 | args.extend(extra_args) 80 | proto = _FowlProtocol(reactor) 81 | transport = await run_service( 82 | reactor, 83 | request, 84 | args=args, 85 | protocol=proto, 86 | ) 87 | return _Fowl(transport, proto) 88 | 89 | 90 | class Echo(Protocol): 91 | def dataReceived(self, data): 92 | self.transport.write(data) 93 | 94 | 95 | class Hello(Protocol): 96 | def connectionMade(self): 97 | self._data = b"" 98 | self.transport.write(b"Hello, world!") 99 | 100 | def dataReceived(self, data): 101 | self._data += data 102 | 103 | 104 | # could use hypothesis to try 'a bunch of ports' but fixed ports seem 105 | # easier to reason about to me 106 | @pytest_twisted.ensureDeferred 107 | async def test_human(reactor, request, wormhole): 108 | """ 109 | """ 110 | f0 = await fowl(reactor, request, "--local", "8000:8008", "--remote", "7000:7007", mailbox=wormhole.url) 111 | m = await f0.protocol.have_line(".*code: (.*)\x1b.*") 112 | code = m.group(1) 113 | print(f"saw code: {code}") 114 | 115 | f1 = await fowl(reactor, request, "--allow-connect", "localhost:8008", "--allow-listen", "7000", code, mailbox=wormhole.url) 116 | 117 | if False: 118 | # XXX can we pull the verifiers out easily? 119 | # verifiers match 120 | m = await f0.protocol.have_line("Verifier: (.*)") 121 | f0_verify = m.group(1) 122 | 123 | m = await f1.protocol.have_line("Verifier: (.*)") 124 | f1_verify = m.group(1) 125 | assert f1_verify == f0_verify, "Verifiers don't match" 126 | 127 | # wait until we see one side listening 128 | while True: 129 | await deferLater(reactor, 0.5, lambda: None) 130 | print("Waiting for at least one listener") 131 | if "🧙" in f0.protocol._data or "🧙" in f1.protocol._data: 132 | print("see one side listening") 133 | break 134 | 135 | print("Making a local connection") 136 | port = await serverFromString(reactor, "tcp:8008:interface=localhost").listen( 137 | Factory.forProtocol(Echo) 138 | ) 139 | request.addfinalizer(lambda:port.stopListening()) 140 | print(" listening on 8008") 141 | 142 | ep1 = clientFromString(reactor, "tcp:localhost:8000") 143 | print(" connecting to 8000") 144 | proto = await ep1.connect(Factory.forProtocol(Hello)) 145 | print(" sending data, awaiting reply") 146 | 147 | for _ in range(5): 148 | await deferLater(reactor, 0.2, lambda: None) 149 | if proto._data == b"Hello, world!": 150 | break 151 | print(f" got {len(proto._data)} bytes reply") 152 | assert proto._data == b"Hello, world!", "Did not see expected echo reply across wormhole" 153 | 154 | 155 | def _get_our_ip(): 156 | s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 157 | s.settimeout(0) 158 | try: 159 | # doesn't even have to be reachable 160 | s.connect(('10.254.254.254', 1)) 161 | IP = s.getsockname()[0] 162 | except Exception: 163 | IP = '127.0.0.1' 164 | finally: 165 | s.close() 166 | return IP 167 | 168 | 169 | @pytest_twisted.ensureDeferred 170 | async def test_non_localhost(reactor, request, wormhole): 171 | """ 172 | """ 173 | # attempt to discover "our" IP address -- so we can attempt to 174 | # connect to it, but not via localhost 175 | ours = _get_our_ip() 176 | f0 = await fowl(reactor, request, "--local", f"127.0.0.1:8111:{ours}:8222", mailbox=wormhole.url) 177 | m = await f0.protocol.have_line(".*code: (.*)\x1b.*") 178 | code = m.group(1) 179 | print(f"saw code: {code}") 180 | 181 | f1 = await fowl(reactor, request, "--allow-connect", f"{ours}:8222", code, mailbox=wormhole.url) 182 | 183 | # wait until we see one side listening 184 | while True: 185 | await deferLater(reactor, 0.5, lambda: None) 186 | print("Waiting for at least one listener") 187 | if "🧙" in f0.protocol._data or "🧙" in f1.protocol._data: 188 | print("see one side listening") 189 | break 190 | 191 | port = await serverFromString(reactor, f"tcp:8222:interface={ours}").listen( 192 | Factory.forProtocol(Echo) 193 | ) 194 | request.addfinalizer(lambda:port.stopListening()) 195 | print(f" listening on 8222:interface={ours}") 196 | 197 | ep1 = clientFromString(reactor, "tcp:localhost:8111") 198 | print(" connecting to 8111") 199 | proto = await ep1.connect(Factory.forProtocol(Hello)) 200 | print(" sending data, awaiting reply") 201 | 202 | for _ in range(5): 203 | await deferLater(reactor, 0.2, lambda: None) 204 | if proto._data == b"Hello, world!": 205 | break 206 | print(f" got {len(proto._data)} bytes reply") 207 | assert proto._data == b"Hello, world!", "Did not see expected echo reply across wormhole" 208 | 209 | 210 | @pytest_twisted.ensureDeferred 211 | async def test_non_localhost_backwards(reactor, request, wormhole): 212 | """ 213 | Same as above test but the 'other way' around 214 | """ 215 | ours = _get_our_ip() 216 | f0 = await fowl(reactor, request, "--remote", f"127.0.0.1:8333:{ours}:8444", "--remote", f"{ours}:8555:8666", mailbox=wormhole.url) 217 | m = await f0.protocol.have_line(".*code: (.*)\x1b.*") 218 | code = m.group(1) 219 | print(f"saw code: {code}") 220 | 221 | f1 = await fowl(reactor, request, "--allow-listen", "8333", "--allow-listen", f"{ours}:8555", code, mailbox=wormhole.url) 222 | 223 | # wait until we see one side listening 224 | while True: 225 | await deferLater(reactor, 0.5, lambda: None) 226 | print("Waiting for at least one listener") 227 | if "🧙" in f0.protocol._data or "🧙" in f1.protocol._data: 228 | print("see one side listening") 229 | break 230 | 231 | port = await serverFromString(reactor, f"tcp:8444:interface={ours}").listen( 232 | Factory.forProtocol(Echo) 233 | ) 234 | request.addfinalizer(lambda:port.stopListening()) 235 | print(f" listening on 8444:interface={ours}") 236 | 237 | ep1 = clientFromString(reactor, "tcp:localhost:8333") 238 | print(" connecting to 8333") 239 | proto = await ep1.connect(Factory.forProtocol(Hello)) 240 | print(" sending data, awaiting reply") 241 | 242 | for _ in range(5): 243 | await deferLater(reactor, 0.2, lambda: None) 244 | if proto._data == b"Hello, world!": 245 | break 246 | print(f" got {len(proto._data)} bytes reply") 247 | assert proto._data == b"Hello, world!", "Did not see expected echo reply across wormhole" 248 | -------------------------------------------------------------------------------- /integration/util.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import attr 4 | import json 5 | from collections import defaultdict 6 | from functools import partial 7 | from io import StringIO 8 | 9 | from attr import define 10 | 11 | import pytest_twisted 12 | 13 | from twisted.internet.interfaces import ITransport 14 | from twisted.internet.protocol import ProcessProtocol 15 | from twisted.internet.error import ProcessExitedAlready 16 | from twisted.internet.defer import Deferred 17 | 18 | from fowl._proto import parse_fowld_output 19 | 20 | 21 | class _MagicTextProtocol(ProcessProtocol): 22 | """ 23 | Internal helper. Monitors all stdout looking for a magic string, 24 | and then .callback()s on self.done or .errback's if the process exits 25 | """ 26 | 27 | def __init__(self, magic_text, print_logs=True): 28 | self.magic_seen = Deferred() 29 | self.exited = Deferred() 30 | self._magic_text = magic_text 31 | self._output = StringIO() 32 | self._print_logs = print_logs 33 | self._stdout_listeners = [] 34 | 35 | def add_stdout_listener(self, listener): 36 | self._stdout_listeners.append(listener) 37 | 38 | def processEnded(self, reason): 39 | if self.magic_seen is not None: 40 | d, self.magic_seen = self.magic_seen, None 41 | d.errback(Exception("Service failed.")) 42 | self.exited.callback(None) 43 | 44 | def childDataReceived(self, childFD, data): 45 | if childFD == 1: 46 | self.out_received(data) 47 | for x in self._stdout_listeners: 48 | x.stdout_received(data) 49 | elif childFD == 2: 50 | self.err_received(data) 51 | else: 52 | ProcessProtocol.childDataReceived(self, childFD, data) 53 | 54 | def out_received(self, data): 55 | """ 56 | Called with output from stdout. 57 | """ 58 | if self._print_logs: 59 | sys.stdout.write(data.decode("utf8")) 60 | self._output.write(data.decode("utf8")) 61 | if self.magic_seen is not None and self._magic_text in self._output.getvalue(): 62 | # print("Saw '{}' in the logs".format(self._magic_text)) 63 | d, self.magic_seen = self.magic_seen, None 64 | d.callback(self) 65 | 66 | def err_received(self, data): 67 | """ 68 | Called when non-JSON lines are received on stderr. 69 | """ 70 | sys.stdout.write(data.decode("utf8")) 71 | 72 | 73 | def run_service( 74 | reactor, 75 | request, 76 | args, 77 | magic_text=None, 78 | cwd=None, 79 | print_logs=True, 80 | protocol=None, 81 | ): 82 | """ 83 | Start a service, and capture the output from the service. 84 | 85 | This will start the service. 86 | 87 | The returned deferred will fire (with the IProcessTransport for 88 | the child) once the given magic text is seeen. 89 | 90 | :param reactor: The reactor to use to launch the process. 91 | :param request: The pytest request object to use for cleanup. 92 | :param magic_text: Text to look for in the logs, that indicate the service 93 | is ready to accept requests. 94 | :param args: The arguments to pass following "python -m ...", approximately. 95 | :param cwd: The working directory of the process. 96 | 97 | :return Deferred[IProcessTransport]: The started process. 98 | """ 99 | if protocol is None: 100 | protocol = _MagicTextProtocol(magic_text, print_logs=print_logs) 101 | saw_magic = protocol.magic_seen 102 | else: 103 | saw_magic = Deferred() 104 | saw_magic.callback(None) 105 | 106 | env = os.environ.copy() 107 | env['PYTHONUNBUFFERED'] = '1' 108 | # if we're not running tests while we have "coverage" installed, 109 | # are we even alive? (that is: not coverage here is not optional) 110 | # shout-out to Ned Batchelder for this tool! 111 | realargs = [sys.executable, "-m", "coverage", "run", "--parallel", "-m"] + args 112 | process = reactor.spawnProcess( 113 | protocol, 114 | sys.executable, 115 | realargs, 116 | path=cwd, 117 | env=env, 118 | ) 119 | request.addfinalizer(partial(_cleanup_service_process, process, protocol.exited)) 120 | return saw_magic.addCallback(lambda ignored: process) 121 | 122 | 123 | def _cleanup_service_process(process, exited): 124 | """ 125 | Terminate the given process with a kill signal (SIGKILL on POSIX, 126 | TerminateProcess on Windows). 127 | 128 | :param process: The `IProcessTransport` representing the process. 129 | :param exited: A `Deferred` which fires when the process has exited. 130 | 131 | :return: After the process has exited. 132 | """ 133 | try: 134 | if process.pid is not None: 135 | # print(f"signaling {process.pid} with TERM") 136 | process.signalProcess('TERM') 137 | # print("signaled, blocking on exit") 138 | pytest_twisted.blockon(exited) 139 | except ProcessExitedAlready: 140 | pass 141 | 142 | 143 | @attr.s 144 | class WormholeMailboxServer: 145 | """ 146 | A locally-running Magic Wormhole mailbox server (on port 4000) 147 | """ 148 | reactor = attr.ib() 149 | process_transport = attr.ib() 150 | url = attr.ib() 151 | 152 | @classmethod 153 | async def create(cls, reactor, request): 154 | args = [ 155 | "twisted", 156 | "wormhole-mailbox", 157 | # note, this tied to "url" below 158 | "--port", "tcp:4000:interface=localhost", 159 | ] 160 | transport = await run_service( 161 | reactor, 162 | request, 163 | args=args, 164 | magic_text="Starting reactor...", 165 | print_logs=False, # twisted json struct-log 166 | ) 167 | return cls( 168 | reactor, 169 | transport, 170 | url="ws://localhost:4000/v1", 171 | ) 172 | 173 | 174 | @define 175 | class _Fowl: 176 | transport: ITransport 177 | protocol: ProcessProtocol 178 | 179 | 180 | class _FowlProtocol(ProcessProtocol): 181 | """ 182 | This speaks to an underlying ``fowl`` sub-process. 183 | """ 184 | 185 | def __init__(self): 186 | # all messages we've received that _haven't_ yet been asked 187 | # for via next_message() 188 | self._messages = [] 189 | # maps str -> list[Deferred]: kind-string to awaiters 190 | self._message_awaits = defaultdict(list) 191 | self.exited = Deferred() 192 | self._data = b"" 193 | 194 | def processEnded(self, reason): 195 | self.exited.callback(None) 196 | 197 | def childDataReceived(self, childFD, data): 198 | if childFD != 1: 199 | print(data.decode("utf8"), end="") 200 | return 201 | 202 | self._data += data 203 | while b'\n' in self._data: 204 | line, self._data = self._data.split(b"\n", 1) 205 | try: 206 | msg = parse_fowld_output(line) 207 | except Exception as e: 208 | print(f"Not JSON: {line}: {e}") 209 | else: 210 | self._maybe_notify(msg) 211 | 212 | def _maybe_notify(self, msg): 213 | type_ = type(msg) 214 | if type_ in self._message_awaits: 215 | notify, self._message_awaits[type_] = self._message_awaits[type_], list() 216 | for d in notify: 217 | d.callback(msg) 218 | else: 219 | self._messages.append(msg) 220 | 221 | def send_message(self, js): 222 | data = json.dumps(js).encode("utf8") + b"\n" 223 | self.transport.write(data) 224 | 225 | def next_message(self, klass): 226 | d = Deferred() 227 | for idx, msg in enumerate(self._messages): 228 | if isinstance(msg, klass): 229 | del self._messages[idx] 230 | d.callback(msg) 231 | return d 232 | self._message_awaits[klass].append(d) 233 | return d 234 | 235 | def all_messages(self, klass=None): 236 | # we _do_ want to make a copy of the list every time 237 | # (so the caller can't "accidentally" mess with our state) 238 | return [ 239 | msg 240 | for msg in self._messages 241 | if klass is None or isinstance(msg, klass) 242 | ] 243 | 244 | 245 | async def fowld(reactor, request, *extra_args, mailbox=None): 246 | """ 247 | Run `fowl` with a given subcommand 248 | """ 249 | 250 | args = [ 251 | "fowl", 252 | ] 253 | if mailbox is not None: 254 | args.extend([ 255 | "--mailbox", mailbox, 256 | ]) 257 | args.extend(extra_args) 258 | proto = _FowlProtocol() 259 | transport = await run_service( 260 | reactor, 261 | request, 262 | args=args, 263 | protocol=proto, 264 | ) 265 | return _Fowl(transport, proto) 266 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling"] 3 | build-backend = "hatchling.build" 4 | 5 | [tool.pytest.ini_options] 6 | filterwarnings = [ 7 | "ignore" 8 | ] 9 | 10 | [tool.ruff] 11 | line-length = 100 12 | [tool.ruff.lint] 13 | ignore = [ 14 | "E712" # at least in policy tests, I want to ensure API returns booleans 15 | ] 16 | 17 | [project] 18 | name = "fowl" 19 | dynamic = ["version"] 20 | description = "Forward Over Wormhole Locally provides bi-directional streaming data over secure and durable Dilated magic-wormhole connections. Secure communication with easy setup." 21 | readme = "README.rst" 22 | license = {file = "LICENSE"} 23 | authors = [ 24 | { name = "meejah", email = "meejah@meejah.ca" }, 25 | ] 26 | requires-python = ">=3.6" 27 | keywords = [ 28 | "cryptography", 29 | "forwarding", 30 | "magic-wormhole", 31 | "private", 32 | ] 33 | classifiers = [ 34 | "Framework :: Twisted", 35 | "Programming Language :: Python :: 3", 36 | "License :: OSI Approved :: MIT License", 37 | ] 38 | dependencies = [ 39 | "setuptools", 40 | "click", 41 | "attrs", 42 | "six", 43 | "msgpack", 44 | "humanize", 45 | "twisted", 46 | "magic-wormhole[dilate] @ git+https://github.com/magic-wormhole/magic-wormhole.git#egg=magic-wormhole[dilate]", 47 | "rich", 48 | ] 49 | 50 | [project.optional-dependencies] 51 | test = [ 52 | "pytest", 53 | "pytest-twisted", 54 | "pytest-cov", 55 | "magic-wormhole-mailbox-server", 56 | "cuvner", 57 | "hypothesis", 58 | "ruff", 59 | ] 60 | dev = [ 61 | "twine", 62 | "sphinx", 63 | "dulwich", 64 | "gpg", # should use isis lovecruft's version? 65 | "pip-tools", 66 | "hatch", 67 | "readme-renderer", 68 | "cuvner", 69 | ] 70 | 71 | [project.scripts] 72 | fowl = "fowl.cli:fowl" 73 | fowld = "fowl.cli:fowld" 74 | 75 | [tool.hatch.metadata] 76 | allow-direct-references = true 77 | 78 | [tool.hatch.version] 79 | path = "src/fowl/__init__.py" 80 | 81 | [tool.hatch.build.targets.sdist] 82 | include = [ 83 | "src", 84 | "docs", 85 | "Makefile", 86 | "README.rst", 87 | "pyproject.toml", 88 | "requirements-pinned.txt", 89 | ] 90 | exclude = [ 91 | "*~", 92 | "*.egg-info*", 93 | ] 94 | 95 | [tool.coverage.run] 96 | branch = true 97 | parallel = true 98 | source_pkgs = ["fowl"] 99 | 100 | ## so my weird coverage problem .. was a "coverage combine" when 101 | ## there's a "leftover" .coverage.* file 102 | ## 103 | ## ..i guess "coverage combine" just combines .coverage.* (and NOT including .coverage ??) -------------------------------------------------------------------------------- /src/fowl/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "24.12.0" 2 | -------------------------------------------------------------------------------- /src/fowl/__main__.py: -------------------------------------------------------------------------------- 1 | from .cli import fowld 2 | 3 | if __name__ == "__main__": 4 | fowld() 5 | -------------------------------------------------------------------------------- /src/fowl/_tui.py: -------------------------------------------------------------------------------- 1 | import curses 2 | import textwrap 3 | import functools 4 | from typing import Optional 5 | from base64 import b16encode # for ping/pong 6 | from os import urandom 7 | 8 | import humanize 9 | 10 | from twisted.internet.task import deferLater 11 | from twisted.internet.defer import ensureDeferred, race 12 | from twisted.internet.stdio import StandardIO 13 | from twisted.protocols.basic import LineReceiver 14 | 15 | from wormhole.errors import LonelyError 16 | 17 | import attr 18 | 19 | from .observer import Next, When 20 | from ._proto import wormhole_from_config, create_fowl 21 | from .messages import ( 22 | Welcome, 23 | CodeAllocated, 24 | PeerConnected, 25 | WormholeClosed, 26 | AllocateCode, 27 | SetCode, 28 | LocalListener, 29 | RemoteListener, 30 | Listening, 31 | RemoteListeningSucceeded, 32 | BytesIn, 33 | BytesOut, 34 | IncomingConnection, 35 | IncomingDone, 36 | IncomingLost, 37 | OutgoingConnection, 38 | OutgoingDone, 39 | WormholeError, 40 | GrantPermission, 41 | Ping, 42 | Pong, 43 | ) 44 | 45 | 46 | @attr.frozen 47 | class Connection: 48 | i: int = 0 49 | o: int = 0 50 | listener_id: str = None # Maybe; Nothing if incoming connection 51 | 52 | 53 | @attr.frozen 54 | class State: 55 | code: Optional[str] = None 56 | connected: bool = False 57 | verifier: Optional[str] = None 58 | listeners: list = attr.Factory(list) 59 | remote_listeners: list = attr.Factory(list) 60 | connections: dict[int, Connection] = attr.Factory(dict) 61 | 62 | @property 63 | def pretty_verifier(self): 64 | # space-ify this, for easier reading 65 | return " ".join( 66 | self.verifier[a:a+4] 67 | for a in range(0, len(self.verifier), 4) 68 | ) 69 | 70 | 71 | async def frontend_tui(reactor, config): 72 | print(f"Connecting: {config.relay_url}") 73 | 74 | @functools.singledispatch 75 | def output_message(msg): 76 | print(f"\b\b\b\bunhandled output: {msg}") 77 | 78 | @output_message.register(Pong) 79 | def _(msg): 80 | print(f"\b\b\b <- Pong({b16encode(msg.ping_id).decode('utf8')}): {msg.time_of_flight}s\n>>> ", end="") 81 | 82 | @output_message.register(WormholeError) 83 | def _(msg): 84 | print(f"\b\b\b\bERROR: {msg.message}") 85 | 86 | @output_message.register(Listening) 87 | def _(msg): 88 | print(f"\b\b\b\bListening: {msg.listen}") 89 | replace_state(attr.evolve(state[0], listeners=state[0].listeners + [msg])) 90 | 91 | @output_message.register(RemoteListeningSucceeded) 92 | def _(msg): 93 | print(f"\b\b\b\bRemote side is listening: {msg.listen}") 94 | replace_state(attr.evolve(state[0], remote_listeners=state[0].remote_listeners + [msg])) 95 | 96 | @output_message.register(IncomingConnection) 97 | def _(msg): 98 | conn = state[0].connections 99 | conn[msg.id] = Connection(0, 0, msg.listener_id) 100 | replace_state(attr.evolve(state[0], connections=conn)) 101 | 102 | @output_message.register(IncomingDone) 103 | def _(msg): 104 | print(f"\b\b\b\bClosed: {msg.id}") 105 | conn = state[0].connections 106 | del conn[msg.id] 107 | replace_state(attr.evolve(state[0], connections=conn)) 108 | 109 | @output_message.register(IncomingLost) 110 | def _(msg): 111 | print(f"\b\b\b\bLost: {msg.id}: {msg.reason}") 112 | conn = state[0].connections 113 | del conn[msg.id] 114 | replace_state(attr.evolve(state[0], connections=conn)) 115 | 116 | @output_message.register(OutgoingConnection) 117 | def _(msg): 118 | conn = state[0].connections 119 | conn[msg.id] = Connection(0, 0, msg.listener_id) 120 | replace_state(attr.evolve(state[0], connections=conn)) 121 | 122 | @output_message.register(OutgoingDone) 123 | def _(msg): 124 | conn = state[0].connections 125 | print(f"\b\b\b\bClosed: {msg.id} from {conn[msg.id].listener_id}") 126 | del conn[msg.id] 127 | replace_state(attr.evolve(state[0], connections=conn)) 128 | 129 | @output_message.register(BytesIn) 130 | def _(msg): 131 | conn = state[0].connections 132 | conn[msg.id] = attr.evolve(conn[msg.id], i=conn[msg.id].i + msg.bytes) 133 | replace_state(attr.evolve(state[0], connections=conn)) 134 | 135 | @output_message.register(BytesOut) 136 | def _(msg): 137 | conn = state[0].connections 138 | conn[msg.id] = attr.evolve(conn[msg.id], o=conn[msg.id].o + msg.bytes) 139 | replace_state(attr.evolve(state[0], connections=conn)) 140 | 141 | @output_message.register(WormholeClosed) 142 | def _(msg): 143 | print(f"Closed({msg.result})...", end="", flush=True) 144 | # close our standard input, so the human can't give us more 145 | # commands -- because we're done 146 | command_reader.transport.loseConnection() 147 | 148 | @output_message.register(Welcome) 149 | def _(msg): 150 | print("\b\b\b\b", end="") 151 | print("Connected.") 152 | if "motd" in msg.welcome: 153 | print(textwrap.fill(msg.welcome["motd"].strip(), 80, initial_indent=" ", subsequent_indent=" ")) 154 | print(">>> ", end="", flush=True) 155 | 156 | fowl_wh = await create_fowl(config, output_message) 157 | 158 | # make into IService? 159 | fowl_wh.start() 160 | 161 | state = [State()] 162 | 163 | def replace_state(new_state): 164 | ##print("replace with", new_state) 165 | ##print(state[0]) 166 | old = state[0] 167 | new_output = "\b\b\b\b" 168 | if new_state.connected and not old.connected: 169 | new_output += "Connected to peer!\n" 170 | if new_state.code and old.code is None: 171 | new_output += "Code: {}\n".format(new_state.code) 172 | if new_state.verifier and old.verifier is None: 173 | new_output += "Verifier: {}\n".format(new_state.pretty_verifier) 174 | for conid, conn in new_state.connections.items(): 175 | b = conn.i + conn.o 176 | if b: 177 | new_output += f"{conid}: {humanize.naturalsize(b)}\n" 178 | if new_output: 179 | print(f"{new_output}>>> ", end="", flush=True) 180 | state[0] = new_state 181 | 182 | @output_message.register(CodeAllocated) 183 | def _(msg): 184 | replace_state(attr.evolve(state[0], code=msg.code)) 185 | 186 | @output_message.register(PeerConnected) 187 | def _(msg): 188 | replace_state(attr.evolve(state[0], connected=True, verifier=msg.verifier)) 189 | 190 | create_stdio = config.create_stdio or StandardIO 191 | command_reader = CommandReader(reactor) 192 | create_stdio(command_reader) 193 | 194 | print(">>> ", end="", flush=True) 195 | while True: 196 | what, result = await race([ 197 | ensureDeferred(command_reader.next_command()), 198 | ensureDeferred(command_reader.when_closed()), 199 | ]) 200 | if what == 0: 201 | cmd_line = result 202 | if cmd_line.strip(): 203 | cmd = cmd_line.decode("utf8").split() 204 | cmd_name = cmd[0] 205 | try: 206 | cmd_fn = commands[cmd_name] 207 | except KeyError: 208 | if cmd_name.strip().lower() == "quit" or cmd_name.strip().lower() == "q": 209 | break 210 | print(f'No such command "{cmd_name}"') 211 | print("Commands: {}".format(" ".join(commands.keys()))) 212 | print("Ctrl-d to quit") 213 | print(">>> ", end="", flush=True) 214 | continue 215 | # XXX should be passing "high level" FowlWormhole thing, not Wormhole direct 216 | await cmd_fn(reactor, fowl_wh, state[0], *cmd[1:]) 217 | else: 218 | print(">>> ", end="", flush=True) 219 | elif what == 1: 220 | break 221 | 222 | print("\nClosing mailbox...", end="", flush=True) 223 | try: 224 | await fowl_wh.stop() 225 | except LonelyError: 226 | pass 227 | print("done.") 228 | 229 | 230 | async def _cmd_help(reactor, wh, state, *args): 231 | """ 232 | Some helpful words 233 | """ 234 | funs = dict() 235 | for name, fn in commands.items(): 236 | try: 237 | funs[fn].append(name) 238 | except KeyError: 239 | funs[fn] = [name] 240 | for fn, aliases in funs.items(): 241 | name = sorted(aliases)[-1] 242 | rest = " ".join(sorted(aliases)[:-1]) 243 | helptext = textwrap.dedent(fn.__doc__) 244 | if helptext: 245 | print(f"{name} ({rest})") 246 | print(textwrap.fill(helptext.strip(), 80, initial_indent=" ", subsequent_indent=" ")) 247 | print() 248 | 249 | 250 | async def _cmd_invite(reactor, wh, state, *args): 251 | """ 252 | Allocate a code (to give to a peer) 253 | """ 254 | if args: 255 | print("No arguments allowed") 256 | return 257 | # XXX fixme no private usage 258 | if state.code is not None: 259 | print(f"Existing code: {state.code}") 260 | else: 261 | wh.command(AllocateCode()) 262 | 263 | 264 | async def _cmd_accept(reactor, wh, state, *args): 265 | """ 266 | Consume an already-allocated code (from a peer) 267 | """ 268 | if len(args) != 1: 269 | print('Require a secret code (e.g. from "invite" on the other side)') 270 | return 271 | if state.code is not None: 272 | print(f"Existing code: {state.code}") 273 | else: 274 | wh.command(SetCode(args[0])) 275 | 276 | 277 | async def _cmd_listen_local(reactor, wh, state, *args): 278 | """ 279 | Listen locally on the given port; connect to the remote side on 280 | the same port (or a custom one if two ports are passed) 281 | """ 282 | try: 283 | port = int(args[0]) 284 | except (ValueError, IndexError): 285 | print("Requires a TCP port, as an integer.") 286 | print("We will listen on this TCP port on localhost, and connect the same") 287 | print("localhost port on the far side. Optionally, a second port may be") 288 | print("specified to use a different far-side port") 289 | return 290 | 291 | if len(args) > 1: 292 | try: 293 | remote_port = int(args[1]) 294 | except ValueError: 295 | print(f"Not port-number: {args[1]}") 296 | return 297 | else: 298 | remote_port = port 299 | 300 | wh.command( 301 | GrantPermission( 302 | listen=[port], 303 | connect=[], 304 | ) 305 | ) 306 | wh.command( 307 | LocalListener( 308 | listen=f"tcp:{port}:interface=localhost", 309 | connect=f"tcp:localhost:{remote_port}", 310 | ) 311 | ) 312 | 313 | 314 | async def _cmd_listen_remote(reactor, wh, state, *args): 315 | """ 316 | Listen on the remote side on the given port; connect back to this 317 | side on the same port (or a custom one if two ports are passed) 318 | """ 319 | try: 320 | remote_port = int(args[0]) 321 | except (ValueError, IndexError): 322 | print("Requires a TCP port, as an integer.") 323 | print("We will listen on this TCP port on the remote side and connect to the same") 324 | print("localhost port on this side. Optionally, a second port may be specified") 325 | print("to use a different local port") 326 | return 327 | 328 | if len(args) > 1: 329 | try: 330 | local_port = int(args[1]) 331 | except ValueError: 332 | print(f"Not port-number: {args[1]}") 333 | return 334 | else: 335 | local_port = remote_port 336 | 337 | wh.command( 338 | GrantPermission( 339 | listen=[], 340 | connect=[local_port], 341 | ) 342 | ) 343 | wh.command( 344 | RemoteListener( 345 | listen=f"tcp:{remote_port}:interface=localhost", 346 | connect=f"tcp:localhost:{local_port}", 347 | ) 348 | ) 349 | 350 | 351 | async def _cmd_allow(reactor, wh, state, *args): 352 | """ 353 | Allow an incoming connection on a particular port 354 | """ 355 | try: 356 | local_port = int(args[0]) 357 | except (ValueError, IndexError): 358 | print("Requires a TCP port, as an integer.") 359 | print("If the other side tries to connect via this port, we will allow it") 360 | return 361 | wh.command( 362 | GrantPermission( 363 | listen=[], 364 | connect=[local_port], 365 | ) 366 | ) 367 | 368 | 369 | async def _cmd_allow_listen(reactor, wh, state, *args): 370 | """ 371 | Allow remote side to listen on a given TCP port. 372 | """ 373 | try: 374 | local_port = int(args[0]) 375 | except (ValueError, IndexError): 376 | print("Requires a TCP port, as an integer.") 377 | print("We will allow the other side to listen on this TCP port") 378 | return 379 | wh.command( 380 | GrantPermission( 381 | listen=[local_port], 382 | connect=[], 383 | ) 384 | ) 385 | 386 | 387 | async def _cmd_ping(reactor, wh, state, *args): 388 | """ 389 | Send a ping (through the Mailbox Server) 390 | """ 391 | ping_id = None 392 | if len(args) != 0: 393 | raise Exception("No argument accepted to ping") 394 | ping_id = urandom(4) 395 | print(f" -> Ping({b16encode(ping_id).decode('utf8')})\n>>> ", end="") 396 | wh.command(Ping(ping_id)) 397 | 398 | 399 | async def _cmd_status(reactor, wh, state, *args): 400 | print("status") 401 | peer = "disconnected" 402 | if state.connected: 403 | peer = "yes" 404 | if state.verifier: 405 | peer += f" verifier={state.verifier}" 406 | else: 407 | print(f" code: {state.code}") 408 | print(f" peer: {peer}") 409 | if state.listeners: 410 | print(" listeners:") 411 | for listener in state.listeners: 412 | print(f" {listener.listener_id.lower()}: {listener.listen} -> {listener.connect}") 413 | if state.remote_listeners: 414 | print(" remote listeners:") 415 | for listener in state.remote_listeners: 416 | print(f" {listener.listener_id}: {listener.connect} <- {listener.listen}") 417 | if state.connections: 418 | print(" connections:") 419 | for conn_id, conn in state.connections.items(): 420 | listener = "" 421 | if conn.listener_id is not None: 422 | listener = f" (via {conn.listener_id})" 423 | print(f" {conn_id}: {conn.i} bytes in / {conn.o} bytes out{listener}") 424 | print(">>> ", end="", flush=True) 425 | 426 | 427 | class CommandReader(LineReceiver): 428 | """ 429 | Wait for incoming commands from the user 430 | """ 431 | delimiter = b"\n" 432 | _next_command = None 433 | _closed = None 434 | reactor = None 435 | 436 | def __init__(self, reactor): 437 | super().__init__() 438 | self.reactor = reactor 439 | self._next_command = Next() 440 | self._closed = When() 441 | 442 | async def next_command(self): 443 | return await self._next_command.next_item() 444 | 445 | async def when_closed(self): 446 | return await self._closed.when_triggered() 447 | 448 | def connectionMade(self): 449 | pass 450 | 451 | def lineReceived(self, line): 452 | self._next_command.trigger(self.reactor, line) 453 | 454 | def connectionLost(self, why): 455 | # beware: triggering with "why" goes to errback chain, not 456 | # what we usually want 457 | self._closed.trigger(self.reactor, None) 458 | 459 | 460 | ## okay, well that is entertaining, but want something quicker to 461 | ## facilitate refactoring, so "REPL-style" it'll be for now? 462 | ## ...but would be great to expand that to have "status" style 463 | ## info "above" the repl stuff, e.g. bandwidth etc 464 | async def curses_frontend_tui(reactor, config): 465 | w = await wormhole_from_config(config) 466 | 467 | # XXX does "curses.wrapper" work with async functions? 468 | ##curses.wrapper(partial(_tui, config, w)) 469 | 470 | try: 471 | stdscr = curses.initscr() 472 | curses.noecho() 473 | curses.cbreak() 474 | curses.curs_set(0) 475 | stdscr.keypad(True) 476 | stdscr.nodelay(True) # non-blocking getch() 477 | 478 | await _tui(reactor, config, w, stdscr) 479 | 480 | finally: 481 | curses.nocbreak() 482 | stdscr.keypad(False) 483 | curses.echo() 484 | curses.endwin() 485 | 486 | 487 | async def sleep(reactor, delay): 488 | await deferLater(reactor, delay, lambda: None) 489 | 490 | 491 | async def _tui(reactor, config, w, stdscr): 492 | stdscr.clear() 493 | 494 | d0 = w.get_welcome() 495 | d1 = w.get_code() 496 | d2 = w.get_versions() 497 | 498 | stdscr.addstr(0, 1, "welcome:") 499 | stdscr.addstr(1, 4, "code:") 500 | stdscr.addstr(2, 0, "versions:") 501 | stdscr.refresh() 502 | 503 | def got_welcome(js): 504 | stdscr.addstr(0, 1, f"welcome: {js}") 505 | stdscr.refresh() 506 | w.allocate_code() 507 | return js 508 | d0.addCallback(got_welcome) 509 | 510 | def got_code(code): 511 | stdscr.addstr(1, 4, f"code: {code}") 512 | stdscr.refresh() 513 | return code 514 | d1.addCallback(got_code) 515 | 516 | def got_versions(versions): 517 | w.dilate() 518 | stdscr.addstr(2, 0, f"versions: {versions}") 519 | stdscr.refresh() 520 | return versions 521 | d2.addCallback(got_versions) 522 | 523 | while True: 524 | # NOTE: it's vital to "wait" or something on the reactor, 525 | # otherwise we're madly busy-looping in curses (only!) and 526 | # nothing else can happen 527 | await sleep(reactor, 0.1) 528 | k = stdscr.getch() 529 | if k != curses.ERR: 530 | if k == ord('q') or k == ord('Q') or k == 27: 531 | break 532 | 533 | try: 534 | await w.close() 535 | except LonelyError: 536 | pass 537 | 538 | 539 | commands = { 540 | "invite": _cmd_invite, 541 | "i": _cmd_invite, 542 | 543 | "accept": _cmd_accept, 544 | "a": _cmd_accept, 545 | 546 | "local": _cmd_listen_local, 547 | "remote": _cmd_listen_remote, 548 | 549 | "allow": _cmd_allow, 550 | "allow-listen": _cmd_allow_listen, 551 | 552 | "ping": _cmd_ping, 553 | "p": _cmd_ping, 554 | 555 | "status": _cmd_status, 556 | "st": _cmd_status, 557 | "s": _cmd_status, 558 | 559 | "help": _cmd_help, 560 | "h":_cmd_help, 561 | "?": _cmd_help, 562 | } 563 | -------------------------------------------------------------------------------- /src/fowl/chicken.py: -------------------------------------------------------------------------------- 1 | # ascii-art graphics used by the default "fowl" terminal visualizers 2 | 3 | default = [ 4 | r""" ,. 5 | (\(\) 6 | ; . > 7 | / (_) """, 8 | r""" ,. 9 | (\(\) 10 | ; o > 11 | / (_) """, 12 | r""" ,. 13 | (\(\) 14 | ; * > 15 | / (_) """, 16 | r""" ,. 17 | (\(\) 18 | ; - > 19 | / (_) """, 20 | r""" ,. 21 | (\(\) 22 | ; ^ > 23 | / (_) """ 24 | ] 25 | 26 | peer = [ 27 | r""" // 28 | o) 37 | (\\_// 38 | (_\_/ """, 39 | r""" // 40 | <*) 41 | (\\_// 42 | (_\_/ """, 43 | r""" // 44 | -*) 45 | (\\_// 46 | (_\_/ """, 47 | ] 48 | -------------------------------------------------------------------------------- /src/fowl/cli.py: -------------------------------------------------------------------------------- 1 | 2 | import click 3 | import pkg_resources 4 | 5 | from twisted.internet.task import react 6 | from twisted.internet.defer import ensureDeferred 7 | 8 | from wormhole.cli.public_relay import ( 9 | RENDEZVOUS_RELAY as PUBLIC_MAILBOX_URL, 10 | ) 11 | 12 | from ._proto import ( 13 | _Config, 14 | forward, 15 | frontend_accept_or_invite, 16 | WELL_KNOWN_MAILBOXES, 17 | ) 18 | from ._tui import frontend_tui 19 | from .messages import ( 20 | LocalListener, 21 | RemoteListener, 22 | ) 23 | from .policy import ( 24 | LocalhostTcpPortsListenPolicy, 25 | LocalhostTcpPortsConnectPolicy, 26 | ArbitraryAddressTcpConnectPolicy, 27 | ArbitraryInterfaceTcpPortsListenPolicy, 28 | is_localhost, 29 | ) 30 | 31 | 32 | WOULD_DO_NOTHING_ERROR = """ 33 | You have requested no listeners and allowed neither listening nor connecting. 34 | This would not do anything useful. 35 | 36 | You should use at least one of: --remote, --local, --allow-listen or --allow-connect 37 | For more information: fowl --help 38 | """ 39 | 40 | 41 | 42 | @click.option( 43 | "--ip-privacy/--clearnet", 44 | default=False, 45 | help="Enable operation over Tor (default is public Internet)", 46 | ) 47 | @click.option( 48 | "--mailbox", 49 | default=PUBLIC_MAILBOX_URL, 50 | help='URL for the mailbox server to use (or "default", "local" or "winden" to use well-known servers)', 51 | metavar="URL or NAME", 52 | ) 53 | @click.option( 54 | "--debug", 55 | default=None, 56 | help="Output wormhole state-machine transitions to the given file", 57 | type=click.File("w", encoding="utf8"), 58 | ) 59 | @click.command() 60 | @click.pass_context 61 | def fowld(ctx, ip_privacy, mailbox, debug): 62 | """ 63 | Forward Over Wormhole, Locally, Daemon 64 | 65 | Low-level daemon to set up and forward streams over Dilated magic 66 | wormhole connections 67 | """ 68 | ctx.obj = _Config( 69 | relay_url=WELL_KNOWN_MAILBOXES.get(mailbox, mailbox), 70 | use_tor=bool(ip_privacy), 71 | debug_file=debug, 72 | # these will be empty; client must activate ports by sending 73 | # messages to allow listening (or connecting) 74 | listen_policy = LocalhostTcpPortsListenPolicy([]), 75 | connect_policy = LocalhostTcpPortsConnectPolicy([]), 76 | ) 77 | def run(reactor): 78 | return ensureDeferred( 79 | forward( 80 | reactor, 81 | ctx.obj, 82 | ) 83 | ) 84 | return react(run) 85 | 86 | 87 | @click.option( 88 | "--debug-messages", 89 | default=None, 90 | type=click.File(mode="w", encoding="utf8"), 91 | help="Save all input/output messages to a file", 92 | ) 93 | @click.option( 94 | "--ip-privacy/--clearnet", 95 | default=False, 96 | help="Enable operation over Tor (default is public Internet)", 97 | ) 98 | @click.option( 99 | "--mailbox", 100 | default=PUBLIC_MAILBOX_URL, 101 | help='URL for the mailbox server to use (or "default" or "winden" to use well-known servers)', 102 | metavar="URL or NAME", 103 | ) 104 | @click.option( 105 | "--debug", 106 | default=None, 107 | help="Output wormhole state-machine transitions to the given file", 108 | type=click.File("w", encoding="utf8"), 109 | ) 110 | @click.option( 111 | "--local", "-L", 112 | multiple=True, 113 | help=( 114 | "Listen locally, connect remotely (accepted multiple times). " 115 | "Unless otherwise specified, (local) bind and (remote) connect addresses are localhost. " 116 | 'For example "127.0.0.1:1234:127.0.0.1:22" is the same as "1234:22" effectively.' 117 | ), 118 | metavar="[bind-address:]listen-port[:remote-address][:connect-port]", 119 | ) 120 | @click.option( 121 | "--remote", "-R", 122 | multiple=True, 123 | help=( 124 | "Listen remotely, connect locally (accepted multiple times) " 125 | "Unless otherwise specified, the (remote) bind and (local) connect addresses are localhost. " 126 | 'For example "127.0.0.1:1234:127.0.0.1:22" is the same as "1234:22" effectively.' 127 | ), 128 | metavar="[remote-bind-address:]listen-port[:local-connect-address][:local-connect-port]", 129 | ) 130 | @click.option( 131 | "--allow-listen", 132 | multiple=True, 133 | help=( 134 | "Accept a connection to this local port. Accepted multiple times. " 135 | "Note that local listeners added via --local are already allowed and do not need this option. " 136 | 'If no interface is specified, "localhost" is assumed.' 137 | ), 138 | metavar="[interface:]listen-port", 139 | ) 140 | @click.option( 141 | "--allow-connect", 142 | multiple=True, 143 | help=( 144 | "Accept a connection to this local port. Accepted multiple times " 145 | 'If no address is specified, "localhost" is assumed.' 146 | ), 147 | metavar="[address:]connect-port", 148 | ) 149 | @click.option( 150 | "--code-length", 151 | default=2, 152 | help="Length of the Wormhole code (if we allocate one)", 153 | ) 154 | @click.option( 155 | "--readme", "-r", 156 | help="Display the full project README", 157 | is_flag=True, 158 | ) 159 | @click.option( 160 | "--interactive", "-i", 161 | help="Run in interactive mode, a human-friendly fowld", 162 | is_flag=True, 163 | ) 164 | @click.argument("code", required=False) 165 | @click.command() 166 | def fowl(ip_privacy, mailbox, debug, allow_listen, allow_connect, local, remote, code_length, code, readme, interactive, debug_messages): 167 | """ 168 | Forward Over Wormhole, Locally 169 | 170 | Bi-directional streaming data over secure and durable Dilated 171 | magic-wormhole connections. 172 | 173 | This frontend is meant for humans -- if you want machine-parsable 174 | data and commands, use fowld (or 'python -m fowl') 175 | 176 | This will create a new session (allocating a fresh code) by 177 | default. To join an existing session (e.g. you've been given a 178 | code) add the code as an (optional) argument on the command-line. 179 | """ 180 | if readme: 181 | display_readme() 182 | return 183 | 184 | local_commands = [ 185 | _specifier_to_tuples(cmd) 186 | for cmd in local 187 | ] 188 | remote_commands = [ 189 | _specifier_to_tuples(cmd) 190 | for cmd in remote 191 | ] 192 | 193 | def to_local(local_interface, local_port, remote_address, remote_port): 194 | return LocalListener( 195 | f"tcp:{local_port}:interface={local_interface}", 196 | f"tcp:{remote_address}:{remote_port}", 197 | ) 198 | 199 | def to_remote(local_interface, local_port, remote_address, remote_port): 200 | return RemoteListener( 201 | f"tcp:{local_port}:interface={local_interface}", 202 | f"tcp:{remote_address}:{remote_port}", 203 | ) 204 | 205 | def to_listen_policy(local_interface, local_port, remote_address, remote_port): 206 | return local_port 207 | 208 | def to_connect_policy(local_interface, local_port, remote_address, remote_port): 209 | return remote_port 210 | 211 | def to_iface_port(allowed): 212 | if ':' in allowed: 213 | iface, port = allowed.split(':', 1) 214 | return iface, _to_port(port) 215 | return "localhost", _to_port(allowed) 216 | 217 | def to_local_port(allowed): 218 | if ':' in allowed: 219 | iface, port = allowed.split(':', 1) 220 | if iface != "localhost": 221 | raise ValueError(f"Non-local interface: {iface}") 222 | return _to_port(port) 223 | return _to_port(allowed) 224 | 225 | def is_local(local_interface, local_port, remote_address, remote_port): 226 | return is_localhost(local_interface) 227 | 228 | def is_local_connect(local_interface, local_port, remote_address, remote_port): 229 | return is_localhost(remote_address) 230 | 231 | if any(not is_local(*cmd) for cmd in local_commands) or \ 232 | any(not is_localhost(to_iface_port(allowed)[0]) for allowed in allow_listen): 233 | listen_policy = ArbitraryInterfaceTcpPortsListenPolicy( 234 | [(iface, port) for iface, port, _, _ in local_commands] + \ 235 | [to_iface_port(allowed) for allowed in allow_listen] 236 | ) 237 | else: 238 | listen_policy = LocalhostTcpPortsListenPolicy( 239 | [to_listen_policy(*cmd) for cmd in local_commands] + 240 | [to_local_port(port) for port in allow_listen] 241 | ) 242 | 243 | if any(not is_local_connect(*cmd) for cmd in remote_commands) or \ 244 | any(not is_localhost(to_iface_port(allowed)[0]) for allowed in allow_connect): 245 | # yes, this says "to_iface_port()" below but they both look 246 | # the same currently: "192.168.1.2:4321" for example 247 | connect_policy = ArbitraryAddressTcpConnectPolicy( 248 | [(addr, port) for _, _, addr, port in remote_commands] + \ 249 | [to_iface_port(allowed) for allowed in allow_connect] 250 | ) 251 | else: 252 | connect_policy = LocalhostTcpPortsConnectPolicy( 253 | [to_connect_policy(*cmd) for cmd in remote_commands] + 254 | [to_local_port(port) for port in allow_connect] 255 | ) 256 | 257 | all_commands = [ 258 | to_local(*t) 259 | for t in local_commands 260 | ] + [ 261 | to_remote(*t) 262 | for t in remote_commands 263 | ] 264 | if not all_commands and not connect_policy and not listen_policy: 265 | raise click.UsageError(WOULD_DO_NOTHING_ERROR) 266 | 267 | cfg = _Config( 268 | relay_url=WELL_KNOWN_MAILBOXES.get(mailbox, mailbox), 269 | use_tor=bool(ip_privacy), 270 | debug_file=debug, 271 | code=code, 272 | code_length=code_length, 273 | commands=all_commands, 274 | listen_policy=listen_policy, 275 | connect_policy=connect_policy, 276 | output_debug_messages=debug_messages, 277 | ) 278 | 279 | if interactive: 280 | return tui(cfg) 281 | 282 | def run(reactor): 283 | return ensureDeferred(frontend_accept_or_invite(reactor, cfg)) 284 | return react(run) 285 | 286 | 287 | def _to_port(arg): 288 | arg = int(arg) 289 | if arg < 1 or arg >= 65536: 290 | raise click.UsageError( 291 | "Ports must be an integer from 1 to 65535" 292 | ) 293 | return arg 294 | 295 | 296 | # XXX FIXME use an @frozen attr, not tuple for returns 297 | def _specifier_to_tuples(cmd): 298 | """ 299 | Parse a local or remote listen/connect specifiers. 300 | 301 | This always returns a 4-tuple of: 302 | - listen interface 303 | - listen port 304 | - connect address 305 | - connect port 306 | 307 | TODO: tests, and IPv6 308 | """ 309 | if '[' in cmd or ']' in cmd: 310 | raise RuntimeError("Have not considered IPv6 parsing yet") 311 | 312 | colons = cmd.count(':') 313 | if colons > 3: 314 | raise ValueError( 315 | f"Too many colons: {colons} > 3" 316 | ) 317 | if colons == 3: 318 | # everything is specified 319 | listen_interface, listen_port, connect_address, connect_port = cmd.split(':') 320 | listen_port = _to_port(listen_port) 321 | connect_port = _to_port(connect_port) 322 | elif colons == 2: 323 | # one of the interface / address is specified, but we're not 324 | # sure which yet 325 | a, b, c = cmd.split(':') 326 | try: 327 | # maybe the first thing is a port 328 | listen_port = _to_port(a) 329 | listen_interface = "localhost" 330 | connect_address = b 331 | connect_port = _to_port(c) 332 | except ValueError: 333 | # no, the first thing is a string, so the connect address 334 | # must be missing 335 | listen_interface = a 336 | listen_port = _to_port(b) 337 | connect_address = "localhost" 338 | connect_port = _to_port(c) 339 | elif colons == 1: 340 | # we only have one split. this could be "interface:port" or "port:port" 341 | a, b = cmd.split(':') 342 | try: 343 | listen_port = _to_port(a) 344 | listen_interface = "localhost" 345 | try: 346 | # the second thing could be a connect address or a 347 | # port 348 | connect_port = _to_port(b) 349 | connect_address = "localhost" 350 | except ValueError: 351 | connect_address = b 352 | connect_port = listen_port 353 | except ValueError: 354 | # okay, first thing isn't a port so it's the listen interface 355 | listen_interface = a 356 | listen_port = connect_port = _to_port(b) 357 | connect_address = "localhost" 358 | else: 359 | # no colons, it's a port and we're "symmetric" 360 | listen_port = connect_port = _to_port(cmd) 361 | listen_interface = "localhost" 362 | connect_address = "localhost" 363 | 364 | # XXX ipv6? 365 | return ( 366 | listen_interface, listen_port, 367 | connect_address, connect_port, 368 | ) 369 | 370 | 371 | def tui(cfg): 372 | """ 373 | Run an interactive text user-interface (TUI) 374 | 375 | Allows one to use a human-readable version of the controller 376 | protocol directly to set up listeners, monitor streams, etc 377 | """ 378 | 379 | def run(reactor): 380 | return ensureDeferred(frontend_tui(reactor, cfg)) 381 | return react(run) 382 | 383 | 384 | def display_readme(): 385 | """ 386 | Display the project README 387 | """ 388 | readme = pkg_resources.resource_string('fowl', '../../README.rst') 389 | # uhm, docutils documentation is confusing as all hell and no good 390 | # examples of "convert this rST string to anything else" .. :/ but 391 | # we should "render" it to text 392 | click.echo_via_pager(readme.decode('utf8')) 393 | 394 | 395 | if __name__ == "__main__": 396 | try: 397 | import coverage 398 | coverage.process_startup() 399 | except ImportError: 400 | pass 401 | fowl() 402 | -------------------------------------------------------------------------------- /src/fowl/daemon.py: -------------------------------------------------------------------------------- 1 | import binascii 2 | import automat 3 | 4 | from .messages import ( 5 | Welcome, 6 | CodeAllocated, 7 | PeerConnected, 8 | SendMessageToPeer, 9 | GotMessageFromPeer, 10 | WormholeClosed, 11 | PleaseCloseWormhole, 12 | ) 13 | 14 | 15 | class FowlDaemon: 16 | """ 17 | This is the core 'fowld' protocol. 18 | 19 | This class MUST NOT do any I/O or any asynchronous things (no 20 | "async def" or Deferred-returning methods). 21 | 22 | There are two main pieces of I/O that we are concerned with here: 23 | - stdin / stdout, line-based JSON messages 24 | - wormhole interactions 25 | 26 | Most users of fowld will see the stdin/stdout message (only), as 27 | that is how we communicate with the outside world. 28 | 29 | Internally (e.g. "tui", "fowl invite", tests) we are concerned 30 | with both kinds of interactions. Especially for tests, sometimes 31 | we want to "not actually interact" with the wormhole (but instead 32 | confirm what this class _wants_ to do). 33 | 34 | Follwing https://sans-io.readthedocs.io/how-to-sans-io.html and 35 | related ideas and posts, instances of this class interact SOLELY 36 | through "input" methods, or FowlCommandMessage (via the "input" 37 | method "command()"). 38 | 39 | The `message_handler` you pass here will get instances of 40 | `FowlOutputMessage` and is expected to "do the I/O" as 41 | appropriate; in the "normal" case this means either dumping a JSON 42 | line to stdout or calling a wormhole method (causing I/O to the 43 | peer or mailbox server). 44 | 45 | On the "input" side, JSON on stdin becomes `FowlCommandMessage` 46 | instances, which are translated into the correct input method to 47 | call. In the "normal" case this means parsing incoming lines with 48 | `parse_fowld_command` and translating that to an appropriate call 49 | to an `@m.input()` decorated method (e.g. got_welcome(), 50 | code_allocated(), etc ..) 51 | 52 | XXX: would it be more clear to just have a .command() and de-multiplex in here? 53 | """ 54 | m = automat.MethodicalMachine() 55 | set_trace = m._setTrace 56 | 57 | def __init__(self, config, message_handler, command_handler): 58 | self._config = config 59 | self._messages = [] # pending plaintext messages to peer 60 | self._verifier = None 61 | self._versions = None 62 | self._message_out = message_handler 63 | self._command_out = command_handler 64 | 65 | def _emit_message(self, msg): 66 | """ 67 | Internal helper to pass a message to our external message handler 68 | (and do something useful on error) 69 | """ 70 | try: 71 | self._message_out(msg) 72 | except Exception as e: 73 | print(f"Error in user code sending {msg}: {e}") 74 | print(type(e), e) 75 | 76 | def _emit_command(self, msg): 77 | """ 78 | Internal helper to pass a command up to our IO handler 79 | """ 80 | self._command_out(msg) 81 | 82 | @m.state(initial=True) 83 | def waiting_code(self): 84 | """ 85 | """ 86 | 87 | @m.state() 88 | def waiting_peer(self): 89 | """ 90 | Do not yet have a peer 91 | """ 92 | 93 | @m.state() 94 | def verifying_version(self): 95 | """ 96 | Does our peer have compatible versions? 97 | """ 98 | 99 | @m.state() 100 | def connected(self): 101 | """ 102 | Normal processing, our peer is connected 103 | """ 104 | 105 | @m.state() 106 | def closing(self): 107 | """ 108 | We have asked to close the wormhole, wait until it is 109 | closed 110 | """ 111 | 112 | @m.state() 113 | def closed(self): 114 | """ 115 | Nothing more to accomplish, the wormhole is closed 116 | """ 117 | 118 | @m.input() 119 | def got_welcome(self, hello): 120 | """ 121 | We have received a 'Welcome' message from the server 122 | """ 123 | 124 | @m.input() 125 | def code_allocated(self, code): 126 | """ 127 | We have a wormhole code (either because it was "set" or because we 128 | allcoated a new one). 129 | """ 130 | # this happens either because we "set-code" or because we 131 | # asked for a new code (i.e. "invite" or "accept") from the 132 | # OUTSIDE (i.e. controlling function) 133 | 134 | @m.input() 135 | def peer_connected(self, verifier: bytes, peer_features: dict): 136 | """ 137 | We have a peer 138 | 139 | :param verifier: a tagged hash of the peers symmetric 140 | key. This should match what our peer sees (users can 141 | verify out-of-band for extra security) 142 | 143 | :param peer_features: arbitrary JSON-able data from the peer, 144 | intended to be used for protocol and other negotiation. A 145 | one-time, at-startup pre-communication mechanism (definitely 146 | before any other messages). Also serves as key-confirmation. 147 | """ 148 | 149 | @m.input() 150 | def version_ok(self, verifier, peer_features): 151 | """ 152 | Our peer version is compatible with ours 153 | """ 154 | 155 | @m.input() 156 | def version_incompatible(self, verifier, peer_features): 157 | """ 158 | We cannot speak with this peer 159 | """ 160 | 161 | @m.input() 162 | def got_message(self, plaintext): 163 | """ 164 | We received a message from our peer 165 | """ 166 | 167 | @m.input() 168 | def send_message(self, plaintext): 169 | """ 170 | Pass a new message to our peer 171 | """ 172 | 173 | @m.input() 174 | def shutdown(self, result): 175 | """ 176 | The wormhole has closed 177 | """ 178 | 179 | @m.output() 180 | def emit_code_allocated(self, code): 181 | self._emit_message(CodeAllocated(code)) 182 | 183 | @m.output() 184 | def emit_peer_connected(self, verifier, peer_features): 185 | """ 186 | """ 187 | self._emit_message( 188 | PeerConnected( 189 | binascii.hexlify(verifier).decode("utf8"), 190 | peer_features, 191 | ) 192 | ) 193 | 194 | @m.output() 195 | def emit_send_message(self, plaintext): 196 | self._emit_message( 197 | SendMessageToPeer( 198 | plaintext, 199 | ) 200 | ) 201 | 202 | @m.output() 203 | def emit_got_message(self, plaintext): 204 | self._emit_message( 205 | GotMessageFromPeer( 206 | plaintext, 207 | ) 208 | ) 209 | 210 | @m.output() 211 | def emit_welcome(self, hello): 212 | self._emit_message( 213 | Welcome(self._config.relay_url, hello) 214 | ) 215 | 216 | @m.output() 217 | def verify_version(self, verifier, peer_features): 218 | """ 219 | Check that our peer supports the right features 220 | """ 221 | features = peer_features.get("fowl", {}).get("features", []) 222 | # note: if we add a feature we will want to do an intersection 223 | # or something and then trigger different behavior depending 224 | # what our peer supports .. for now there's only one thing, 225 | # and it MUST be supported 226 | from ._proto import SUPPORTED_FEATURES # FIXME 227 | if set(features) == set(SUPPORTED_FEATURES): 228 | self.version_ok(verifier, peer_features) 229 | else: 230 | self.version_incompatible(verifier, peer_features) 231 | 232 | @m.output() 233 | def queue_message(self, plaintext): 234 | self._message_queue.append(plaintext) 235 | 236 | @m.output() 237 | def send_queued_messages(self): 238 | to_send, self._messages = self._messages, [] 239 | for m in to_send: 240 | self.emit_send_message(m) 241 | 242 | @m.output() 243 | def emit_shutdown(self, result): 244 | self._emit_message( 245 | WormholeClosed(result) 246 | ) 247 | 248 | @m.output() 249 | def emit_close_wormhole(self): 250 | self._emit_command( 251 | PleaseCloseWormhole("versions are incompatible") # XXX hardcoded bad 252 | ) 253 | 254 | waiting_code.upon( 255 | code_allocated, 256 | enter=waiting_peer, 257 | outputs=[emit_code_allocated], 258 | ) 259 | waiting_code.upon( 260 | send_message, 261 | enter=waiting_code, 262 | outputs=[queue_message] 263 | ) 264 | waiting_code.upon( 265 | got_welcome, 266 | enter=waiting_code, 267 | outputs=[emit_welcome] 268 | ) 269 | waiting_code.upon( 270 | shutdown, 271 | enter=closed, 272 | outputs=[emit_shutdown] 273 | ) 274 | 275 | waiting_peer.upon( 276 | send_message, 277 | enter=waiting_peer, 278 | outputs=[queue_message] 279 | ) 280 | waiting_peer.upon( 281 | got_welcome, 282 | enter=waiting_peer, 283 | outputs=[emit_welcome] 284 | ) 285 | waiting_peer.upon( 286 | got_message, 287 | enter=waiting_peer, 288 | outputs=[emit_got_message] 289 | ) 290 | waiting_peer.upon( 291 | peer_connected, 292 | enter=verifying_version, 293 | outputs=[verify_version] 294 | ) 295 | verifying_version.upon( 296 | version_ok, 297 | enter=connected, 298 | outputs=[emit_peer_connected, send_queued_messages], 299 | ) 300 | verifying_version.upon( 301 | version_incompatible, 302 | enter=closing, 303 | outputs=[emit_close_wormhole], 304 | ) 305 | waiting_peer.upon( 306 | shutdown, 307 | enter=closed, 308 | outputs=[emit_shutdown] 309 | ) 310 | 311 | connected.upon( 312 | send_message, 313 | enter=connected, 314 | outputs=[emit_send_message] 315 | ) 316 | connected.upon( 317 | got_message, 318 | enter=connected, 319 | outputs=[emit_got_message] 320 | ) 321 | connected.upon( 322 | shutdown, 323 | enter=closed, 324 | outputs=[emit_shutdown] 325 | ) 326 | 327 | closing.upon( 328 | shutdown, 329 | enter=closed, 330 | outputs=[emit_shutdown] 331 | ) 332 | # XXX there's no notification to go from "connected" to 333 | # "waiting_peer" -- because Dilation will silently "do the right 334 | # thing" (so we don't need to). But it would be nice to tell the 335 | # user if we're between "generations" or whatever 336 | -------------------------------------------------------------------------------- /src/fowl/messages.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | from attrs import frozen 3 | 4 | 5 | class FowlOutputMessage: 6 | """ 7 | An information message from fowld to the controller 8 | """ 9 | 10 | 11 | class FowlCommandMessage: 12 | """ 13 | A command from the controller to fowld 14 | """ 15 | 16 | 17 | class FowlInternalControl: 18 | "A message from the state-machine to outside, basically?" 19 | pass 20 | 21 | 22 | # if we had ADT / Union types, these would both be that -- is this as 23 | # close as we can get in Python? 24 | 25 | 26 | @frozen 27 | class Welcome(FowlOutputMessage): 28 | """ 29 | We have connected to the Mailbox Server and received the 30 | Welcome message. 31 | """ 32 | url: str # server address 33 | # open-ended information from the server 34 | welcome: dict 35 | 36 | 37 | @frozen 38 | class WormholeClosed(FowlOutputMessage): 39 | """ 40 | The wormhole has been terminated. 41 | """ 42 | result: str 43 | 44 | 45 | @frozen 46 | class AllocateCode(FowlCommandMessage): 47 | """ 48 | Create a fresh code on the server 49 | """ 50 | length: Optional[int] = None 51 | 52 | 53 | @frozen 54 | class SetCode(FowlCommandMessage): 55 | """ 56 | Give a code we know to the server 57 | """ 58 | code: str 59 | 60 | 61 | @frozen 62 | class CodeAllocated(FowlOutputMessage): 63 | """ 64 | The secret wormhole code has been determined 65 | """ 66 | code: str 67 | 68 | 69 | @frozen 70 | class PeerConnected(FowlOutputMessage): 71 | """ 72 | We have evidence that the peer has connected 73 | """ 74 | # hex-encoded 32-byte hash output (should match other peer) 75 | verifier: str 76 | versions: dict 77 | 78 | 79 | @frozen 80 | class GrantPermission(FowlCommandMessage): 81 | """ 82 | Grant additional listen or connection privileges. Both are lists 83 | of valid ports between 1 and 65535 inclusive. 84 | """ 85 | listen: list[int] 86 | connect: list[int] 87 | 88 | 89 | @frozen 90 | class DangerDisablePermissionCheck(FowlCommandMessage): 91 | """ 92 | DANGER: allow listening or connecting to anything. Can be 93 | dangerous, you must know the implications. 94 | """ 95 | 96 | 97 | @frozen 98 | class LocalListener(FowlCommandMessage): 99 | """ 100 | We wish to open a local listener. 101 | """ 102 | listen: str # Twisted server-type endpoint string 103 | connect: str # Twisted client-type endpoint string 104 | 105 | 106 | @frozen 107 | class RemoteListener(FowlCommandMessage): 108 | """ 109 | We wish to open a listener on the peer. 110 | """ 111 | listen: str # Twisted server-type endpoint string 112 | connect: str # Twisted client-type endpoint string 113 | 114 | 115 | @frozen 116 | class Ping(FowlCommandMessage): 117 | ping_id: int 118 | 119 | 120 | @frozen 121 | class Listening(FowlOutputMessage): 122 | """ 123 | We have opened a local listener. 124 | 125 | Any connections to this listener will result in a subchannel and a 126 | connect on the other side (to "connected_endpoint"). This message 127 | may result from a LocalListener or a RemoteListener command. This 128 | message will always appear on the side that's actually listening. 129 | """ 130 | listener_id: str # random identifier 131 | listen: str # Twisted server-type endpoint string 132 | connect: str # Twisted client-type endpoint string 133 | 134 | 135 | @frozen 136 | class RemoteListeningFailed(FowlOutputMessage): 137 | """ 138 | We have failed to open a listener on the remote side. 139 | """ 140 | listen: str # Twisted server-type endpoint string 141 | reason: str 142 | 143 | 144 | @frozen 145 | class RemoteListeningSucceeded(FowlOutputMessage): 146 | """ 147 | The remote peer suceeded at fulfilling our listen request. 148 | """ 149 | listener_id: str # arbitrary identifier 150 | listen: str # Twisted server-type endpoint string 151 | connect: str # Twisted client-type endpoint string 152 | 153 | 154 | @frozen 155 | class RemoteConnectFailed(FowlOutputMessage): 156 | """ 157 | Our peer could not connect 158 | """ 159 | id: int 160 | reason: str 161 | 162 | 163 | @frozen 164 | class OutgoingConnection(FowlOutputMessage): 165 | """ 166 | Something has connected to one of our listeners (and we are making 167 | an outgoing subchannel to the other peer). 168 | """ 169 | id: int 170 | endpoint: str # connection to here on far side 171 | listener_id: str # which listener this is from 172 | 173 | 174 | @frozen 175 | class OutgoingLost(FowlOutputMessage): 176 | """ 177 | We have lost one of our connections 178 | """ 179 | id: int 180 | reason: str 181 | 182 | 183 | @frozen 184 | class OutgoingDone(FowlOutputMessage): 185 | """ 186 | We have lost one of our connections 187 | """ 188 | id: int 189 | 190 | 191 | @frozen 192 | class IncomingConnection(FowlOutputMessage): 193 | """ 194 | The other side is requesting we open a connection 195 | """ 196 | id: int 197 | endpoint: str 198 | listener_id: str 199 | 200 | 201 | @frozen 202 | class IncomingLost(FowlOutputMessage): 203 | """ 204 | We have lost one of our connections 205 | """ 206 | id: int 207 | reason: str 208 | 209 | 210 | @frozen 211 | class IncomingDone(FowlOutputMessage): 212 | """ 213 | An incoming connection has ended successfully 214 | """ 215 | id: int 216 | 217 | 218 | @frozen 219 | class BytesIn(FowlOutputMessage): 220 | id: int 221 | bytes: int 222 | 223 | 224 | @frozen 225 | class BytesOut(FowlOutputMessage): 226 | id: int 227 | bytes: int 228 | 229 | 230 | @frozen 231 | class WormholeError(FowlOutputMessage): 232 | message: str 233 | 234 | 235 | @frozen 236 | class PleaseCloseWormhole(FowlInternalControl): 237 | reason: str 238 | 239 | 240 | @frozen 241 | class Pong(FowlOutputMessage): 242 | ping_id: int 243 | time_of_flight: float 244 | 245 | 246 | #XXX these aren't really used; why state-machine has them? 247 | @frozen 248 | class SendMessageToPeer(FowlOutputMessage): 249 | message: str 250 | 251 | 252 | @frozen 253 | class GotMessageFromPeer(FowlOutputMessage): 254 | message: str 255 | -------------------------------------------------------------------------------- /src/fowl/observer.py: -------------------------------------------------------------------------------- 1 | from attr import define, Factory 2 | from twisted.internet.defer import Deferred, succeed 3 | 4 | 5 | _UNSET = object() 6 | 7 | 8 | class Framer: 9 | """ 10 | Takes a stream of bytes and produces 'messages' from them, 11 | triggering an underlying Next()-style observable. 12 | 13 | Only produces a single message per reactor 'tick'. If there is 14 | 'already' a subsequent message availble, we use a callLater(0, 15 | ...) to produce it and check further. 16 | 17 | (This is necessary so that something doing an async iteration has 18 | a chance to "do other work" .. e.g. call .next_message() again 19 | ... before any more messages are delivered XXX demo of this problem?) 20 | """ 21 | 22 | def __init__(self, reactor): ## just does LineReceiver for now, find_a_frame): 23 | self._reactor = reactor 24 | self._data = b"" 25 | self._next = Next() 26 | 27 | def next_message(self): 28 | """ 29 | :return Awaitable: a new Deferred that fires when a complete, 30 | as-yet undelivered message has arrived. 31 | """ 32 | return self._next.next_item() 33 | 34 | def data_received(self, data): 35 | self._data += data 36 | self._maybe_deliver_messages() 37 | 38 | def _find_frame(self, data): 39 | """ 40 | hard-coded to LineReceiver, could make more general 41 | 42 | :returns: 2-tuple of the message and remaining data. if 43 | "message" is None, there was no complete message yet and 44 | all data is returned 45 | """ 46 | if b"\n" in data: 47 | return data.split(b"\n", 1) 48 | return (None, data) 49 | 50 | def _maybe_deliver_messages(self): 51 | msg, self._data = self._find_frame(self._data) 52 | if msg is not None: 53 | self._next.trigger(self._reactor, msg.decode("utf8")) 54 | if self._find_frame(self._data)[0] is not None: 55 | self._reactor.callLater(0, self._maybe_deliver_messages) 56 | 57 | 58 | @define 59 | class Next: 60 | """ 61 | An observerable event that can by async-ly listened for and 62 | triggered multiple times. 63 | 64 | Used for implementing the a ``next_thing()`` style of method. 65 | """ 66 | 67 | _awaiters: list = Factory(list) 68 | _unheard_result: object = _UNSET 69 | 70 | def next_item(self): 71 | """ 72 | :return Awaitable: a new Deferred that fires when this observable 73 | triggered. This will always be 'in the future' even if we've 74 | triggered more than zero times already. 75 | """ 76 | if self._unheard_result is not _UNSET: 77 | d = succeed(self._unheard_result) 78 | self._unheard_result = None 79 | else: 80 | d = Deferred() 81 | self._awaiters.append(d) 82 | return d 83 | 84 | def trigger(self, reactor, result): 85 | """ 86 | Triggers all current observers and resets them to the empty list. 87 | """ 88 | listeners, self._awaiters = self._awaiters, [] 89 | if listeners: 90 | for d in listeners: 91 | reactor.callLater(0, d.callback, result) 92 | else: 93 | self._unheard_result = result 94 | 95 | 96 | @define 97 | class When: 98 | """ 99 | An observerable event that can be async-ly listened for and 100 | triggers exactly once. 101 | 102 | Used for implementing the a ``when_thing()`` style of method. 103 | """ 104 | 105 | _awaiters: list = Factory(list) 106 | _result: object = _UNSET 107 | 108 | def when_triggered(self): 109 | """ 110 | :return Awaitable: a new Deferred that fires when this observable 111 | triggered. This maybe be 'right now' if we already have a result 112 | """ 113 | if self._result is not _UNSET: 114 | d = succeed(self._result) 115 | else: 116 | d = Deferred() 117 | self._awaiters.append(d) 118 | return d 119 | 120 | def trigger(self, reactor, result): 121 | """ 122 | Triggers all current observers and resets them to the empty list. 123 | """ 124 | assert self._result is _UNSET, "Can only trigger it once" 125 | listeners, self._awaiters = self._awaiters, [] 126 | self._result = result 127 | for d in listeners: 128 | reactor.callLater(0, d.callback, result) 129 | 130 | 131 | @define 132 | class Accumulate: 133 | """ 134 | An observerable event that can by async-ly listened for and 135 | triggered multiple times, with a per-event 'size' of item to 136 | collect (as observed via len() calls). 137 | 138 | Used for implementing the a ``next_message(size=123)`` style of method. 139 | """ 140 | 141 | _results: object 142 | _awaiters: list = Factory(list) 143 | 144 | def next_item(self, reactor, size): 145 | """ 146 | :return Awaitable: a new Deferred that fires when this observable 147 | triggered. This will always be 'in the future' even if we've 148 | 149 | triggered more than zero times already. 150 | """ 151 | d = Deferred() 152 | self._awaiters.append((size,d)) 153 | self._examine_results(reactor) 154 | return d 155 | 156 | def some_results(self, reactor, result): 157 | """ 158 | Append these results. If this gives us enough results to notify 159 | current listeners, we do. 160 | """ 161 | self._results += result 162 | self._examine_results(reactor) 163 | 164 | def _examine_results(self, reactor): 165 | if not self._awaiters: 166 | return 167 | size, d = self._awaiters[0] 168 | if len(self._results) >= size: 169 | self._awaiters.pop(0) 170 | self._results, result = self._results[size:], self._results[:size] 171 | reactor.callLater(0, d.callback, result) 172 | -------------------------------------------------------------------------------- /src/fowl/policy.py: -------------------------------------------------------------------------------- 1 | 2 | import ipaddress 3 | 4 | from attr import frozen 5 | from zope.interface import Interface, implementer 6 | from twisted.internet.endpoints import ( 7 | TCP4ServerEndpoint, 8 | TCP6ServerEndpoint, 9 | TCP4ClientEndpoint, 10 | TCP6ClientEndpoint, 11 | ) 12 | 13 | 14 | class IClientListenPolicy(Interface): 15 | """ 16 | A way to ask which endpoints are acceptable to listen upon for a client. 17 | """ 18 | 19 | def can_listen(self, endpoint) -> bool: 20 | """ 21 | :returns: True if the given IStreamServerEndpoint is acceptable to 22 | this policy 23 | """ 24 | 25 | 26 | class IClientConnectPolicy(Interface): 27 | """ 28 | Ask what endpoints are acceptable to connect on 29 | """ 30 | 31 | def can_connect(self, endpoint) -> bool: 32 | """ 33 | :returns: True if the given IStreamClientEndpoint is acceptable to 34 | this policy 35 | """ 36 | 37 | 38 | # XXX if i'm offline, "localhost" doesn't work (with ip_address()) -- when _does_ it work, and why? 39 | # XXX with radios off entirely, i'm seeing "" (empty string) as the addr here 40 | def is_localhost(addr: str) -> bool: 41 | if addr.strip() == "": 42 | return False 43 | if addr == "localhost": 44 | return True 45 | ip = ipaddress.ip_address(addr) 46 | return ip.is_loopback 47 | 48 | 49 | @implementer(IClientListenPolicy) 50 | class LocalhostAnyPortsListenPolicy: 51 | """ 52 | Accepts any port as long as the interface is a local one (i.e. ::1 53 | or localhost or 127.0.0.1/8) according to the "ipaddress" library. 54 | """ 55 | def can_listen(self, endpoint) -> bool: 56 | if isinstance(endpoint, (TCP6ServerEndpoint, TCP4ServerEndpoint)): 57 | return is_localhost(endpoint._interface) 58 | return False 59 | 60 | 61 | @implementer(IClientListenPolicy) 62 | @frozen 63 | class LocalhostTcpPortsListenPolicy(LocalhostAnyPortsListenPolicy): 64 | # which ports we will accept 65 | ports: list[int] 66 | 67 | def can_listen(self, endpoint) -> bool: 68 | if super().can_listen(endpoint): 69 | # if we're here, parent has checked types too 70 | if endpoint._port in self.ports: 71 | return True 72 | return False 73 | 74 | def __bool__(self): 75 | return self.ports != [] 76 | 77 | 78 | @implementer(IClientListenPolicy) 79 | @frozen 80 | class ArbitraryInterfaceTcpPortsListenPolicy: 81 | # interface, port pairs we accept 82 | listeners: list[tuple] 83 | 84 | def can_listen(self, endpoint) -> bool: 85 | if isinstance(endpoint, (TCP6ServerEndpoint, TCP4ServerEndpoint)): 86 | iface = endpoint._interface 87 | port = endpoint._port 88 | for allowed_iface, allowed_port in self.listeners: 89 | if is_localhost(allowed_iface) and is_localhost(endpoint._interface): 90 | if port == allowed_port: 91 | return True 92 | if iface == allowed_iface: 93 | if port == allowed_port: 94 | return True 95 | return False 96 | 97 | def __bool__(self): 98 | return self.listeners != [] 99 | 100 | 101 | @implementer(IClientListenPolicy) 102 | class AnyListenPolicy: 103 | """ 104 | Accepts any listener at all. DANGER. 105 | """ 106 | def can_listen(self, endpoint) -> bool: 107 | return True 108 | 109 | 110 | @implementer(IClientConnectPolicy) 111 | class LocalhostAnyPortsConnectPolicy: 112 | """ 113 | Accepts any port as long as the interface is a local one (i.e. ::1 114 | or localhost or 127.0.0.1/8) according to the "ipaddress" library. 115 | """ 116 | def can_connect(self, endpoint) -> bool: 117 | if isinstance(endpoint, (TCP6ClientEndpoint, TCP4ClientEndpoint)): 118 | return is_localhost(endpoint._host) 119 | return False 120 | 121 | 122 | @implementer(IClientListenPolicy) 123 | @frozen 124 | class ArbitraryAddressTcpConnectPolicy: 125 | # interface, port pairs we accept 126 | connecters: list[tuple] 127 | 128 | def can_connect(self, endpoint) -> bool: 129 | if isinstance(endpoint, (TCP6ClientEndpoint, TCP4ClientEndpoint)): 130 | addr, port = endpoint._host, endpoint._port 131 | for allowed_addr, allowed_port in self.connecters: 132 | if addr == allowed_addr: 133 | if port == allowed_port: 134 | return True 135 | return False 136 | 137 | 138 | @implementer(IClientConnectPolicy) 139 | class AnyConnectPolicy: 140 | """ 141 | Accepts any connection at all. DANGER. 142 | """ 143 | def can_connect(self, endpoint) -> bool: 144 | return True 145 | 146 | 147 | @implementer(IClientConnectPolicy) 148 | @frozen 149 | class LocalhostTcpPortsConnectPolicy(LocalhostAnyPortsConnectPolicy): 150 | # which ports we will accept 151 | ports: list[int] 152 | 153 | def can_connect(self, endpoint) -> bool: 154 | if super().can_connect(endpoint): 155 | # if we're here, parent has checked types too 156 | if endpoint._port in self.ports: 157 | return True 158 | return False 159 | 160 | def __bool__(self): 161 | return self.ports != [] 162 | 163 | -------------------------------------------------------------------------------- /src/fowl/status.py: -------------------------------------------------------------------------------- 1 | import time 2 | import json 3 | import functools 4 | from typing import Dict, Optional, Callable 5 | 6 | import attrs 7 | 8 | from wormhole._status import ConnectionStatus 9 | 10 | from fowl.messages import BytesIn, BytesOut, OutgoingConnection, OutgoingDone, OutgoingLost, Listening, Welcome, PeerConnected, RemoteListeningSucceeded, WormholeClosed, CodeAllocated, IncomingConnection, IncomingDone, IncomingLost, GotMessageFromPeer, FowlOutputMessage 11 | 12 | 13 | @attrs.define 14 | class Subchannel: 15 | endpoint: str 16 | listener_id: str 17 | i: list 18 | o: list 19 | 20 | 21 | @attrs.define 22 | class Listener: 23 | listen: str 24 | connect: str 25 | remote: bool 26 | 27 | 28 | @attrs.define 29 | class FowlStatus: 30 | url: Optional[str] = None 31 | mailbox_connection: Optional[ConnectionStatus] = None 32 | welcome: dict = {} 33 | code: Optional[str] = None 34 | verifier: Optional[str] = None 35 | closed: Optional[str] = None # closed status, "happy", "lonely" etc 36 | subchannels: Dict[str, Subchannel] = {} 37 | listeners: Dict[str, Listener] = {} 38 | time_provider: Callable[[], float] = time.time 39 | on_message: Optional[Callable[[FowlOutputMessage], None]] = None 40 | peer_closing: bool = False 41 | we_closing: bool = False 42 | 43 | def __attrs_post_init__(self): 44 | @functools.singledispatch 45 | def on_message(msg): 46 | print(f"unhandled: {msg}") 47 | 48 | @on_message.register(Welcome) 49 | def _(msg): 50 | self.url = msg.url 51 | self.welcome = msg.welcome 52 | self.closed = None 53 | 54 | @on_message.register(CodeAllocated) 55 | def _(msg): 56 | self.code = msg.code 57 | 58 | @on_message.register(PeerConnected) 59 | def _(msg): 60 | self.verifier = msg.verifier 61 | 62 | @on_message.register(GotMessageFromPeer) 63 | def _(msg): 64 | d = json.loads(msg.message) 65 | print(f"peer: {d}") 66 | if "closing" in d: 67 | self.peer_closing = True 68 | 69 | @on_message.register(WormholeClosed) 70 | def _(msg): 71 | self.closed = msg.result 72 | 73 | @on_message.register(Listening) 74 | def _(msg): 75 | self.listeners[msg.listener_id] = Listener(msg.listen, msg.connect, False) 76 | 77 | @on_message.register(RemoteListeningSucceeded) 78 | def _(msg): 79 | self.listeners[msg.listener_id] = Listener(msg.listen, msg.connect, True) 80 | 81 | @on_message.register(BytesIn) 82 | def _(msg): 83 | self.subchannels[msg.id].i.insert(0, (msg.bytes, self.time_provider())) 84 | 85 | @on_message.register(BytesOut) 86 | def _(msg): 87 | self.subchannels[msg.id].o.insert(0, (msg.bytes, self.time_provider())) 88 | 89 | @on_message.register(IncomingConnection) 90 | def _(msg): 91 | self.subchannels[msg.id] = Subchannel(msg.endpoint, msg.listener_id, [], []) 92 | 93 | @on_message.register(IncomingDone) 94 | def _(msg): 95 | #out = humanize.naturalsize(sum([b for b, _ in subchannels[msg.id].o])) 96 | #in_ = humanize.naturalsize(sum([b for b, _ in subchannels[msg.id].i])) 97 | #print(f"{msg.id} closed: {out} out, {in_} in") 98 | del self.subchannels[msg.id] 99 | 100 | @on_message.register(IncomingLost) 101 | def _(msg): 102 | del self.subchannels[msg.id] 103 | 104 | @on_message.register(OutgoingConnection) 105 | def _(msg): 106 | self.subchannels[msg.id] = Subchannel(msg.endpoint, msg.listener_id, [], []) 107 | 108 | @on_message.register(OutgoingDone) 109 | def _(msg): 110 | #out = humanize.naturalsize(sum([b for b, _ in subchannels[msg.id].o])) 111 | #in_ = humanize.naturalsize(sum([b for b, _ in subchannels[msg.id].i])) 112 | #print(f"{msg.id} closed: {out} out, {in_} in") 113 | del self.subchannels[msg.id] 114 | 115 | @on_message.register(OutgoingLost) 116 | def _(msg): 117 | del self.subchannels[msg.id] 118 | 119 | self.on_message = on_message 120 | -------------------------------------------------------------------------------- /src/fowl/test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meejah/fowl/7cdc2b51bb541d7700f435c3497ab13a6f15646d/src/fowl/test/__init__.py -------------------------------------------------------------------------------- /src/fowl/test/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import pytest_twisted 3 | 4 | from twisted.internet.defer import ensureDeferred 5 | 6 | from .util import WormholeMailboxServer 7 | 8 | 9 | @pytest.fixture(scope='session') 10 | def reactor(): 11 | # this is a fixture in case we might want to try different 12 | # reactors for some reason. 13 | from twisted.internet import reactor as _reactor 14 | return _reactor 15 | 16 | 17 | @pytest.fixture(scope='session') 18 | def mailbox(reactor, request): 19 | """ 20 | A global wormhole mailbox server instance, running on localhost 21 | 22 | It's often considered 'better practice' to test things like this 23 | without involving 'actual networking', but wormhole doesn't come 24 | with test tools in that shape so we'll 'suffer' with the pains of 25 | actual, localhost networking. 26 | """ 27 | 28 | return pytest_twisted.blockon( 29 | ensureDeferred( 30 | WormholeMailboxServer.create( 31 | reactor, 32 | request, 33 | ) 34 | ) 35 | ) 36 | 37 | -------------------------------------------------------------------------------- /src/fowl/test/test_cli.py: -------------------------------------------------------------------------------- 1 | 2 | import pytest_twisted 3 | 4 | #from twisted.internet.protocol import ProtocolBase 5 | from zope.interface import implementer 6 | from twisted.internet.interfaces import IProcessProtocol 7 | from twisted.internet.protocol import ProcessProtocol 8 | from twisted.internet.endpoints import serverFromString, clientFromString 9 | from twisted.internet.task import deferLater 10 | from hypothesis.strategies import integers, sampled_from, one_of, ip_addresses 11 | from hypothesis import given 12 | import click 13 | import sys 14 | import os 15 | import signal 16 | from fowl.observer import When, Framer 17 | from fowl.test.util import ServerFactory, ClientFactory 18 | from fowl.cli import _to_port, _specifier_to_tuples 19 | 20 | 21 | @implementer(IProcessProtocol) 22 | class CollectStreams(ProcessProtocol): 23 | 24 | def __init__(self, reactor): 25 | self._reactor = reactor 26 | self._streams = { 27 | 1: b"", 28 | 2: b"", 29 | } 30 | self._done = When() 31 | self._lines = Framer(reactor) 32 | 33 | def when_done(self): 34 | return self._done.when_triggered() 35 | 36 | def next_line(self): 37 | return self._lines.next_message() 38 | 39 | def childDataReceived(self, fd, data): 40 | self._streams[fd] += data 41 | if fd == 1: 42 | self._lines.data_received(data) 43 | 44 | def processExited(self, reason): 45 | pass #reason == Failure 46 | 47 | def processEnded(self, reason): 48 | self._done.trigger(self._reactor, reason) 49 | 50 | 51 | # maybe Hypothesis better, via strategies.binary() ? 52 | @pytest_twisted.ensureDeferred 53 | async def test_happy_path(reactor, request, mailbox): 54 | """ 55 | One side invites, other accepts. 56 | 57 | Some commands are executed. 58 | Improvement: let Hypothesis make up commands, order, etc (how to assert?) 59 | """ 60 | 61 | print("Starting invite side", os.environ.get("COVERAGE_PROCESS_STARTUP", "no startup")) 62 | 63 | invite_proto = CollectStreams(reactor) 64 | invite = reactor.spawnProcess( 65 | invite_proto, 66 | sys.executable, 67 | [ 68 | "python", "-u", "-m", "fowl.cli", 69 | "--mailbox", mailbox.url, 70 | # redundant "--allow-connect", "2121", 71 | "--remote", "2222:2121", 72 | ], 73 | env=os.environ, 74 | ) 75 | request.addfinalizer(lambda:invite.signalProcess(signal.SIGKILL)) 76 | 77 | import re 78 | x = re.compile(b"code: (.*)\x1b") 79 | code = None 80 | while not code: 81 | await deferLater(reactor, 1, lambda: None) 82 | #print(invite_proto._streams) 83 | if m := x.search(invite_proto._streams[1]): 84 | code = m.group(1).decode("utf8") 85 | 86 | print(f"Detected code: {code}") 87 | 88 | accept_proto = CollectStreams(reactor) 89 | accept = reactor.spawnProcess( 90 | accept_proto, 91 | sys.executable, 92 | [ 93 | "python", "-u", "-m", "fowl.cli", 94 | "--mailbox", mailbox.url, 95 | "--allow-listen", "2222", 96 | code, 97 | ], 98 | env=os.environ, 99 | ) 100 | request.addfinalizer(lambda:accept.signalProcess(signal.SIGKILL)) 101 | 102 | print("Starting accept side") 103 | 104 | while True: 105 | if "🧙".encode("utf8") in invite_proto._streams[1]: 106 | print("invite side listening") 107 | break 108 | elif "🧙".encode("utf8") in accept_proto._streams[1]: 109 | print("accept side listening") 110 | break 111 | await deferLater(reactor, 0.5, lambda: None) 112 | 113 | # now that they are connected, and one side is listening -- we can 114 | # ourselves listen on the "connect" port and connect on the 115 | # "listen" port -- that is, listen on 2121 (where there is no 116 | # listener) and connect on 2222 (where this test is listening) 117 | 118 | listener = ServerFactory(reactor) 119 | await serverFromString(reactor, "tcp:2121:interface=localhost").listen(listener) # returns server_port 120 | 121 | client = clientFromString(reactor, "tcp:localhost:2222") 122 | client_proto = await client.connect(ClientFactory(reactor)) 123 | server = await listener.next_client() 124 | 125 | datasize = 1234 126 | data = os.urandom(datasize) 127 | 128 | client_proto.send(data) 129 | msg = await server.next_message(len(data)) 130 | assert msg == data, "Incorrect data transfer" 131 | 132 | print("done") 133 | 134 | 135 | @given(integers(min_value=1, max_value=65535)) 136 | def test_helper_to_port(port): 137 | assert(_to_port(port) == port) 138 | assert(_to_port(str(port)) == port) 139 | 140 | 141 | @given(one_of(integers(max_value=0), integers(min_value=65536))) 142 | def test_helper_to_port_invalid(port): 143 | try: 144 | _to_port(port) 145 | assert False, "Should raise exception" 146 | except click.UsageError: 147 | pass 148 | 149 | 150 | @given(integers(min_value=1, max_value=65535)) 151 | def test_specifiers_one_port(port): 152 | cmd = f"{port}" 153 | assert _specifier_to_tuples(cmd) == ("localhost", port, "localhost", port) 154 | 155 | 156 | @given( 157 | integers(min_value=1, max_value=65535), 158 | integers(min_value=1, max_value=65535), 159 | ) 160 | def test_specifiers_two_ports(port0, port1): 161 | cmd = f"{port0}:{port1}" 162 | assert _specifier_to_tuples(cmd) == ("localhost", port0, "localhost", port1) 163 | 164 | 165 | @given( 166 | integers(min_value=1, max_value=65535), 167 | integers(min_value=1, max_value=65535), 168 | ip_addresses(v=4), # do not support IPv6 yet 169 | ) 170 | def test_specifiers_two_ports_one_ip(port0, port1, ip): 171 | if ip.version == 4: 172 | cmd = f"{ip}:{port0}:{port1}" 173 | else: 174 | cmd = f"[{ip}]:{port0}:{port1}" 175 | assert _specifier_to_tuples(cmd) == (str(ip), port0, "localhost", port1) 176 | 177 | 178 | @given( 179 | integers(min_value=1, max_value=65535), 180 | integers(min_value=1, max_value=65535), 181 | ip_addresses(v=4), # do not support IPv6 yet 182 | ip_addresses(v=4), # do not support IPv6 yet 183 | ) 184 | def test_specifiers_two_ports_two_ips(port0, port1, ip0, ip1): 185 | cmd = f"{ip0}:{port0}:{ip1}:{port1}" 186 | assert _specifier_to_tuples(cmd) == (str(ip0), port0, str(ip1), port1) 187 | 188 | 189 | @given( 190 | integers(min_value=1, max_value=65535), 191 | integers(min_value=1, max_value=65535), 192 | ip_addresses(v=6), 193 | ip_addresses(v=6), 194 | sampled_from([True, False]), 195 | ) 196 | def test_specifiers_unsupported_v6(port0, port1, ip0, ip1, wrap): 197 | if wrap: 198 | cmd = f"[{ip0}]:{port0}:[{ip1}]:{port1}" 199 | else: 200 | cmd = f"{ip0}:{port0}:{ip1}:{port1}" 201 | try: 202 | assert _specifier_to_tuples(cmd) == (str(ip0), port0, str(ip1), port1) 203 | except RuntimeError: 204 | pass 205 | except ValueError: 206 | pass 207 | -------------------------------------------------------------------------------- /src/fowl/test/test_commands.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from hypothesis.strategies import one_of, integers, lists, sampled_from, builds, text 4 | from hypothesis import given 5 | 6 | 7 | from fowl._proto import parse_fowld_command, fowld_command_to_json 8 | 9 | 10 | def command_messages(): 11 | from fowl import messages 12 | return [ 13 | (cls, command_class_to_arg_generators(cls)) 14 | for cls in [getattr(messages, nm) for nm in dir(messages)] 15 | if type(cls) is type and issubclass(cls, messages.FowlCommandMessage) and cls != messages.FowlCommandMessage 16 | ] 17 | 18 | 19 | def command_class_to_arg_generators(cls): 20 | from fowl import messages 21 | return { 22 | messages.AllocateCode: { 23 | "length": integers(min_value=1, max_value=32), 24 | }, 25 | messages.SetCode: { 26 | "code": text(), 27 | }, 28 | messages.BytesIn: { 29 | "id": integers(), 30 | "bytes": integers(min_value=1), 31 | }, 32 | messages.BytesOut: { 33 | "id": integers(), 34 | "bytes": integers(min_value=1), 35 | }, 36 | messages.DangerDisablePermissionCheck: { 37 | }, 38 | messages.LocalListener: { 39 | "listen": local_server_endpoints(), 40 | "connect": local_client_endpoints(), 41 | }, 42 | messages.RemoteListener: { 43 | "listen": local_server_endpoints(), 44 | "connect": local_client_endpoints(), 45 | }, 46 | messages.GrantPermission: { 47 | "listen": port_lists(), 48 | "connect": port_lists(), 49 | }, 50 | messages.Ping: { 51 | "ping_id": text(), # should really be "base16-encoded 4-bytes of binary" 52 | }, 53 | }[cls] 54 | 55 | 56 | def ports(): 57 | return integers(min_value=1, max_value=65535) 58 | 59 | 60 | def port_lists(): 61 | return lists(ports()) 62 | 63 | 64 | def local_server_endpoints(): 65 | return sampled_from([ 66 | "tcp:1234:interface=localhost", 67 | ]) 68 | 69 | 70 | def local_client_endpoints(): 71 | return sampled_from([ 72 | "tcp:localhost:1234", 73 | ]) 74 | 75 | 76 | all_commands = { 77 | k: kwargs 78 | for k, kwargs in command_messages() 79 | } 80 | 81 | 82 | def commands(): 83 | return one_of([ 84 | builds(k, **kwargs) 85 | for k, kwargs in all_commands.items() 86 | ]) 87 | 88 | 89 | 90 | @given(commands()) 91 | def test_roundtrip(og_cmd): 92 | """ 93 | Let Hypothesis play with a bunch of round-trip tests for command 94 | serialization 95 | """ 96 | parsed_cmd = parse_fowld_command(json.dumps(fowld_command_to_json(og_cmd))) 97 | assert parsed_cmd == og_cmd, "Command mismatch" 98 | -------------------------------------------------------------------------------- /src/fowl/test/test_forward.py: -------------------------------------------------------------------------------- 1 | import os 2 | import random 3 | import string 4 | import json 5 | from io import StringIO 6 | 7 | from attr import define 8 | 9 | import pytest 10 | import pytest_twisted 11 | 12 | from twisted.internet.task import deferLater 13 | from twisted.internet.defer import ensureDeferred, CancelledError 14 | from twisted.internet.protocol import Protocol 15 | from twisted.internet.endpoints import serverFromString, clientFromString 16 | 17 | from fowl._proto import _Config, forward 18 | from fowl.observer import Accumulate 19 | from fowl.test.util import ServerFactory, ClientFactory 20 | from fowl.policy import AnyConnectPolicy, AnyListenPolicy 21 | 22 | 23 | # XXX ultimately we might want a "TestingWormhole" object or something 24 | # to put into wormhole proper. 25 | # It would be in-memory and hook up all protocols to .. itself? 26 | 27 | 28 | def create_wormhole_factory(): 29 | 30 | def stream_of_valid_codes(): 31 | for number in range(1, 1000): 32 | code = "{}-{}-{}".format( 33 | number, 34 | random.choice(string.ascii_letters), 35 | random.choice(string.ascii_letters), 36 | ) 37 | yield code 38 | 39 | wormholes = [] 40 | codes = stream_of_valid_codes() 41 | 42 | async def memory_wormhole(cfg): 43 | print("memory wormhole", cfg) 44 | 45 | @define 46 | class Endpoint: 47 | connects: list = [] 48 | listens: list = [] 49 | 50 | async def connect(self, addr): 51 | print("connect", addr) 52 | return self.connects.pop(0) 53 | 54 | def listen(self, factory): 55 | print("listen", factory) 56 | ear = self.listens.pop(0) 57 | return ear(factory) 58 | 59 | @define 60 | class Wormhole: 61 | code: str = None 62 | control_ep: Endpoint = Endpoint() 63 | connect_ep: Endpoint = Endpoint() 64 | listen_ep: Endpoint = Endpoint() 65 | 66 | async def get_welcome(self): 67 | return { 68 | "testing": "this is a testing wormhole", 69 | } 70 | 71 | def allocate_code(self, words): 72 | self.code = next(codes) 73 | return self.code 74 | 75 | async def get_code(self): 76 | return self.code 77 | 78 | async def get_unverified_key(self): 79 | return b"0" * 32 80 | 81 | async def get_verifier(self): 82 | return b"x" * 32 83 | 84 | def dilate(self): 85 | return (self.control_ep, self.connect_ep, self.listen_ep) 86 | 87 | w = Wormhole() 88 | wormholes.append(w) 89 | return w 90 | return memory_wormhole 91 | 92 | 93 | async def sleep(reactor, t): 94 | await deferLater(reactor, t, lambda: None) 95 | 96 | 97 | @pytest_twisted.ensureDeferred 98 | async def find_message(reactor, config, kind=None, timeout=10, show_error=True): 99 | """ 100 | Await a message of particular kind in the stdout of config 101 | 102 | :param bool show_error: when True, immediately print any 103 | "kind=error" messages encountered 104 | """ 105 | for _ in range(timeout): 106 | messages = [ 107 | json.loads(line) 108 | for line in config.stdout.getvalue().split("\n") 109 | if line 110 | ] 111 | for msg in messages: 112 | if msg["kind"] == kind: 113 | return msg 114 | if msg["kind"] == "error": 115 | print(f"error: {msg['message']}") 116 | await sleep(reactor, 1) 117 | if False: 118 | print("no '{}' yet: {}".format(kind, " ".join([m.get("kind", "") for m in messages]))) 119 | raise RuntimeError( 120 | f"Waited {timeout}s for message of kind={kind}" 121 | ) 122 | 123 | 124 | class FakeStandardIO(object): 125 | def __init__(self, proto, reactor, messages): 126 | self.disconnecting = False ## XXX why? this is in normal one? 127 | self.proto = proto 128 | self.reactor = reactor 129 | self.messages = messages 130 | self.proto.makeConnection(self) 131 | for msg in messages: 132 | assert isinstance(msg, bytes), "messages must be bytes" 133 | self.proto.dataReceived(msg) 134 | 135 | 136 | def ignore_cancel(f): 137 | if f.trap(CancelledError): 138 | return None 139 | return f 140 | 141 | 142 | # maybe Hypothesis better, via strategies.binary() ? 143 | @pytest_twisted.ensureDeferred 144 | @pytest.mark.parametrize("datasize", [2**6])#range(2**6, 2**17, 2**15)) 145 | @pytest.mark.parametrize("who", [True, False]) 146 | async def test_forward(reactor, request, mailbox, datasize, who): 147 | 148 | stdios = [ 149 | None, 150 | None, 151 | ] 152 | 153 | def create_stdin0(proto, reactor=None): 154 | stdios[0] = FakeStandardIO(proto, reactor, messages=[]) 155 | return stdios[0] 156 | 157 | def create_stdin1(proto, reactor=None): 158 | stdios[1] = FakeStandardIO(proto, reactor, messages=[]) 159 | return stdios[1] 160 | 161 | config0 = _Config( 162 | relay_url=mailbox.url, 163 | use_tor=False, 164 | create_stdio=create_stdin0, 165 | stdout=StringIO(), 166 | listen_policy=AnyListenPolicy(), 167 | connect_policy=AnyConnectPolicy(), 168 | ) 169 | # note: would like to get rid of this ensureDeferred, but it 170 | # doesn't start "running" the coro until we do this... 171 | d0 = ensureDeferred(forward(reactor, config0)) 172 | d0.addErrback(ignore_cancel) 173 | msg = await find_message(reactor, config0, kind="welcome") 174 | stdios[0].proto.dataReceived( 175 | json.dumps({ 176 | "kind": "allocate-code", 177 | }).encode("utf8") + b"\n" 178 | ) 179 | 180 | msg = await find_message(reactor, config0, kind="code-allocated") 181 | assert 'code' in msg, "Missing code" 182 | 183 | config1 = _Config( 184 | relay_url=mailbox.url, 185 | use_tor=False, 186 | create_stdio=create_stdin1, 187 | stdout=StringIO(), 188 | code=msg["code"], 189 | listen_policy=AnyListenPolicy(), 190 | connect_policy=AnyConnectPolicy(), 191 | ) 192 | 193 | d1 = ensureDeferred(forward(reactor, config1)) 194 | d1.addErrback(ignore_cancel) 195 | msg = await find_message(reactor, config1, kind="welcome") 196 | stdios[1].proto.dataReceived( 197 | json.dumps({ 198 | "kind": "set-code", 199 | "code": config1.code, 200 | }).encode("utf8") + b"\n" 201 | ) 202 | 203 | await find_message(reactor, config0, kind="peer-connected") 204 | await find_message(reactor, config1, kind="peer-connected") 205 | 206 | class Server(Protocol): 207 | _message = Accumulate(b"") 208 | 209 | def dataReceived(self, data): 210 | self._message.some_results(reactor, data) 211 | 212 | async def next_message(self, expected_size): 213 | return await self._message.next_item(reactor, expected_size) 214 | 215 | def send(self, data): 216 | self.transport.write(data) 217 | 218 | 219 | class Client(Protocol): 220 | _message = Accumulate(b"") 221 | 222 | def dataReceived(self, data): 223 | self._message.some_results(reactor, data) 224 | 225 | async def next_message(self, expected_size): 226 | return await self._message.next_item(reactor, expected_size) 227 | 228 | def send(self, data): 229 | self.transport.write(data) 230 | 231 | listener = ServerFactory(reactor) 232 | server_port = await serverFromString(reactor, "tcp:1111").listen(listener) 233 | 234 | # both sides are connected -- now we can issue a "remote listen" 235 | # request 236 | stdios[0].proto.dataReceived( 237 | json.dumps({ 238 | "kind": "local", 239 | "listen": "tcp:7777", 240 | "connect": "tcp:localhost:1111", 241 | }).encode("utf8") + b"\n" 242 | ) 243 | msg = await find_message(reactor, config0, kind="listening") 244 | 245 | # if we do 'too many' test-cases debian complains about 246 | # "twisted.internet.error.ConnectBindError: Couldn't bind: 24: Too 247 | # many open files." 248 | # gc.collect() doesn't fix it. 249 | client = clientFromString(reactor, "tcp:localhost:7777") # NB: same port as in "kind=local" message! 250 | client_proto = await client.connect(ClientFactory(reactor)) 251 | server = await listener.next_client() 252 | 253 | def cleanup(): 254 | d0.cancel() 255 | d1.cancel() 256 | client_proto.transport.loseConnection() 257 | server.transport.loseConnection() 258 | server_port.stopListening() 259 | request.addfinalizer(cleanup) 260 | 261 | data = os.urandom(datasize) 262 | if who: 263 | client_proto.send(data) 264 | msg = await server.next_message(len(data)) 265 | else: 266 | server.send(data) 267 | msg = await client_proto.next_message(len(data)) 268 | who = not who 269 | assert msg == data, "Incorrect data transfer" 270 | 271 | 272 | @pytest_twisted.ensureDeferred 273 | @pytest.mark.parametrize("datasize", [2**6])##range(2**6, 2**17, 2**15)) 274 | @pytest.mark.parametrize("who", [True, False]) 275 | @pytest.mark.parametrize("wait_peer", [True, False]) 276 | async def test_drawrof(reactor, request, mailbox, datasize, who, wait_peer): 277 | 278 | stdios = [ 279 | None, 280 | None, 281 | ] 282 | 283 | def create_stdin0(proto, reactor=None): 284 | stdios[0] = FakeStandardIO(proto, reactor, messages=[]) 285 | return stdios[0] 286 | 287 | def create_stdin1(proto, reactor=None): 288 | stdios[1] = FakeStandardIO(proto, reactor, messages=[]) 289 | return stdios[1] 290 | 291 | config0 = _Config( 292 | relay_url=mailbox.url, 293 | use_tor=False, 294 | create_stdio=create_stdin0, 295 | stdout=StringIO(), 296 | listen_policy=AnyListenPolicy(), 297 | connect_policy=AnyConnectPolicy(), 298 | ) 299 | # note: would like to get rid of this ensureDeferred, but it 300 | # doesn't start "running" the coro until we do this... 301 | d0 = ensureDeferred(forward(reactor, config0)) 302 | d0.addErrback(ignore_cancel) 303 | msg = await find_message(reactor, config0, kind="welcome") 304 | 305 | # when connected, issue a "open listener" to one side 306 | stdios[0].proto.dataReceived( 307 | json.dumps({ 308 | "kind": "allocate-code", 309 | }).encode("utf8") + b"\n" 310 | ) 311 | 312 | msg = await find_message(reactor, config0, kind="code-allocated") 313 | assert 'code' in msg, "Missing code" 314 | 315 | config1 = _Config( 316 | relay_url=mailbox.url, 317 | use_tor=False, 318 | create_stdio=create_stdin1, 319 | stdout=StringIO(), 320 | code=msg["code"], 321 | listen_policy=AnyListenPolicy(), 322 | connect_policy=AnyConnectPolicy(), 323 | ) 324 | d1 = ensureDeferred(forward(reactor, config1)) 325 | d1.addErrback(ignore_cancel) 326 | msg = await find_message(reactor, config1, kind="welcome") 327 | 328 | # now we can set the code on this side 329 | stdios[1].proto.dataReceived( 330 | json.dumps({ 331 | "kind": "set-code", 332 | "code": config1.code, 333 | }).encode("utf8") + b"\n" 334 | ) 335 | 336 | listener = ServerFactory(reactor) 337 | server_port = await serverFromString(reactor, "tcp:3333:interface=localhost").listen(listener) 338 | request.addfinalizer(server_port.stopListening) 339 | 340 | # whether we explicitly wait for our peer, the underlying fowl 341 | # code should "do the right thing" if we just start issuing 342 | # listen/etc commands 343 | if wait_peer: 344 | print("Explicitly awaiting peers") 345 | msg = await find_message(reactor, config0, kind="peer-connected") 346 | msg = await find_message(reactor, config1, kind="peer-connected") 347 | print("Both sides have a peer") 348 | else: 349 | print("Not waiting for peer") 350 | 351 | # both sides are connected -- now we can issue a "remote listen" 352 | # request 353 | stdios[0].proto.dataReceived( 354 | json.dumps({ 355 | "kind": "remote", 356 | "listen": "tcp:8888", 357 | "connect": "tcp:localhost:3333", 358 | }).encode("utf8") + b"\n" 359 | ) 360 | 361 | msg = await find_message(reactor, config1, kind="listening") 362 | print(f"Listening: {msg}") 363 | 364 | # if we do 'too many' test-cases debian complains about 365 | # "twisted.internet.error.ConnectBindError: Couldn't bind: 24: Too 366 | # many open files." 367 | # gc.collect() doesn't fix it. 368 | client = clientFromString(reactor, "tcp:localhost:8888") # NB: same as remote-endpoint 369 | client_proto = await client.connect(ClientFactory(reactor)) 370 | server = await listener.next_client() 371 | 372 | def cleanup(): 373 | d0.cancel() 374 | d1.cancel() 375 | server.transport.loseConnection() 376 | pytest_twisted.blockon(ensureDeferred(server.when_closed())) 377 | request.addfinalizer(cleanup) 378 | 379 | data = os.urandom(datasize) 380 | if who: 381 | print("client sending") 382 | client_proto.send(data) 383 | msg = await server.next_message(len(data)) 384 | else: 385 | print("server sending") 386 | server.send(data) 387 | msg = await client_proto.next_message(len(data)) 388 | assert msg == data, "Incorrect data transfer" 389 | -------------------------------------------------------------------------------- /src/fowl/test/test_policy.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import ipaddress 4 | from hypothesis.strategies import ip_addresses, one_of, integers, lists 5 | from hypothesis import given, assume 6 | 7 | from twisted.internet.endpoints import TCP4ServerEndpoint, TCP6ServerEndpoint 8 | 9 | from fowl.policy import LocalhostAnyPortsListenPolicy, LocalhostTcpPortsListenPolicy 10 | 11 | 12 | def ip_address_to_listener(reactor, port, ipaddr): 13 | """ 14 | Convert an ipaddress.IPv4Address or IPv6Address to an 15 | TCP4ServerEndpoint (or TCP6ServerEndpoint) 16 | """ 17 | if isinstance(ipaddr, ipaddress.IPv4Address): 18 | return TCP4ServerEndpoint(reactor, port, interface=str(ipaddr)) 19 | elif isinstance(ipaddr, ipaddress.IPv6Address): 20 | return TCP6ServerEndpoint(reactor, port, interface=str(ipaddr)) 21 | raise ValueError(f"Unknown ipaddress: {ipaddr}") 22 | 23 | 24 | @given( 25 | integers(1, 65536), # any port we might care about 26 | one_of( 27 | ip_addresses(network="127.0.0.0/8"), 28 | ip_addresses(network="::1"), 29 | # these are "link-local" but I believe different meaning from "localhost"? 30 | # ip_addresses(network="fe80::/64"), 31 | # also what about "ipv4-mapped addresses that are actually localhost? possible? worth it? 32 | ) 33 | ) 34 | def test_policy_any_acceptable(reactor, port, ipaddr): 35 | """ 36 | LocalhostAnyPortsPolicy allows all valid localhost style addresses 37 | we might like 38 | """ 39 | endpoint = ip_address_to_listener(reactor, port, ipaddr) 40 | policy = LocalhostAnyPortsListenPolicy() 41 | assert policy.can_listen(endpoint) == True, "Listen on a localhost port" 42 | 43 | 44 | @given( 45 | integers(1, 65536), # any port we might care about 46 | ip_addresses(), 47 | ) 48 | def test_policy_any_bad(reactor, port, ipaddr): 49 | """ 50 | LocalhostAnyPortsPolicy disallows all IP addresses that are NOT 51 | localhost 52 | """ 53 | assume(not ipaddress.ip_address(ipaddr).is_loopback) 54 | endpoint = ip_address_to_listener(reactor, port, ipaddr) 55 | policy = LocalhostAnyPortsListenPolicy() 56 | assert policy.can_listen(endpoint) == False, "Should only allow loopback addresses" 57 | 58 | 59 | @given( 60 | integers(1, 65536), # any port we might care about 61 | lists(integers(1, 65536)), # allowed ports in our policy 62 | ip_addresses(), 63 | ) 64 | def test_policy_specific_ports(reactor, port, allowed_ports, ipaddr): 65 | """ 66 | LocalhostAnyPortsPolicy disallows all IP addresses that are NOT 67 | localhost 68 | """ 69 | is_local = ipaddress.ip_address(ipaddr).is_loopback 70 | is_allowed = port in allowed_ports 71 | endpoint = ip_address_to_listener(reactor, port, ipaddr) 72 | expected_result = True if is_local and is_allowed else False 73 | 74 | policy = LocalhostTcpPortsListenPolicy(allowed_ports) 75 | assert policy.can_listen(endpoint) == expected_result, f"what port={endpoint._port} if={endpoint._interface} {expected_result} {port} {allowed_ports} {ipaddr} {policy.can_listen(endpoint)}" 76 | -------------------------------------------------------------------------------- /src/fowl/test/util.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | from io import ( 4 | StringIO, 5 | ) 6 | from functools import partial 7 | 8 | import attr 9 | 10 | from twisted.internet.protocol import ProcessProtocol, Protocol, Factory 11 | from twisted.internet.defer import ( 12 | Deferred, 13 | ) 14 | from twisted.internet.error import ( 15 | ProcessExitedAlready, 16 | ) 17 | import pytest_twisted 18 | 19 | from fowl.observer import Accumulate, Next, When 20 | 21 | 22 | class _MagicTextProtocol(ProcessProtocol): 23 | """ 24 | Internal helper. 25 | 26 | Monitors all stdout looking for a magic string, and then 27 | .callback()s on self.done and .errback's if the process exits 28 | """ 29 | 30 | def __init__(self, magic_text, log_function): 31 | self.magic_seen = Deferred() 32 | self.exited = Deferred() 33 | self._magic_text = magic_text 34 | self._output = StringIO() 35 | self._log_function = log_function 36 | 37 | def processEnded(self, reason): 38 | if self.magic_seen is not None: 39 | d, self.magic_seen = self.magic_seen, None 40 | d.errback(Exception("Service failed.")) 41 | self.exited.callback(None) 42 | 43 | def childDataReceived(self, childFD, data): 44 | if childFD == 1: 45 | self.out_received(data) 46 | elif childFD == 2: 47 | self.err_received(data) 48 | else: 49 | ProcessProtocol.childDataReceived(self, childFD, data) 50 | 51 | def out_received(self, data): 52 | """ 53 | Called with output from stdout. 54 | """ 55 | self._output.write(data.decode("utf8")) 56 | if self._log_function: 57 | self._log_function(data.decode("utf8")) 58 | if self.magic_seen is not None and self._magic_text in self._output.getvalue(): 59 | print("Saw '{}' in the logs".format(self._magic_text)) 60 | d, self.magic_seen = self.magic_seen, None 61 | d.callback(self) 62 | 63 | def err_received(self, data): 64 | """ 65 | Output on stderr 66 | """ 67 | sys.stdout.write(data.decode("utf8")) 68 | 69 | 70 | def run_service( 71 | reactor, 72 | request, 73 | magic_text, 74 | executable, 75 | args, 76 | cwd=None, 77 | log_collector=print, 78 | ): 79 | """ 80 | Start a service, and capture the output from the service 81 | 82 | This will start the service, and the returned deferred will fire with 83 | the process, once the given magic text is seeen. 84 | 85 | :param reactor: The reactor to use to launch the process. 86 | :param request: The pytest request object to use for cleanup. 87 | :param magic_text: Text to look for in the logs, that indicate the service 88 | is ready to accept requests. 89 | :param executable: The executable to run. 90 | :param args: The arguments to pass to the process. 91 | :param cwd: The working directory of the process. 92 | 93 | :return Deferred[IProcessTransport]: The started process. 94 | """ 95 | protocol = _MagicTextProtocol(magic_text, log_collector) 96 | 97 | env = os.environ.copy() 98 | env['PYTHONUNBUFFERED'] = '1' 99 | for k, v in env.items(): 100 | if 'COV' in k: 101 | print(k, v) 102 | process = reactor.spawnProcess( 103 | protocol, 104 | executable, 105 | args, 106 | path=cwd, 107 | env=env, 108 | ) 109 | request.addfinalizer(partial(_cleanup_service_process, process, protocol.exited)) 110 | return protocol.magic_seen.addCallback(lambda ignored: process) 111 | 112 | 113 | def _cleanup_service_process(process, exited): 114 | """ 115 | Terminate the given process with a kill signal (SIGKILL on POSIX, 116 | TerminateProcess on Windows). 117 | 118 | :param process: The `IProcessTransport` representing the process. 119 | :param exited: A `Deferred` which fires when the process has exited. 120 | 121 | :return: After the process has exited. 122 | """ 123 | try: 124 | if process.pid is not None: 125 | print("signaling {} with TERM".format(process.pid)) 126 | process.signalProcess('TERM') 127 | print("signaled, blocking on exit") 128 | pytest_twisted.blockon(exited) 129 | print("exited, goodbye") 130 | except ProcessExitedAlready: 131 | pass 132 | 133 | 134 | @attr.s 135 | class WormholeMailboxServer: 136 | """ 137 | A locally-running Magic Wormhole mailbox server 138 | """ 139 | reactor = attr.ib() 140 | process_transport = attr.ib() 141 | url = attr.ib() 142 | logs = attr.ib() 143 | 144 | @classmethod 145 | async def create(cls, reactor, request): 146 | args = [ 147 | sys.executable, 148 | "-m", 149 | "twisted", 150 | "wormhole-mailbox", 151 | # note, this tied to "url" below 152 | "--port", "tcp:4000:interface=localhost", 153 | ] 154 | logs = list() 155 | transport = await run_service( 156 | reactor, 157 | request, 158 | magic_text="Starting reactor...", 159 | executable=sys.executable, 160 | args=args, 161 | log_collector=lambda d: logs.append(d), 162 | ) 163 | # note: run_service adds a finalizer 164 | return cls( 165 | reactor, 166 | transport, 167 | "ws://localhost:4000/v1", 168 | logs, 169 | ) 170 | 171 | 172 | # some helpers for local connections 173 | 174 | 175 | class Server(Protocol): 176 | 177 | def __init__(self): 178 | self._message = Accumulate(b"") 179 | self._done = When() 180 | 181 | async def when_closed(self): 182 | return await self._done.when_triggered() 183 | 184 | async def next_message(self, expected_size): 185 | return await self._message.next_item(self.factory.reactor, expected_size) 186 | 187 | def send(self, data): 188 | self.transport.write(data) 189 | 190 | def dataReceived(self, data): 191 | self._message.some_results(self.factory.reactor, data) 192 | 193 | def connectionLost(self, reason): 194 | self._done.trigger(self.factory.reactor, None) 195 | 196 | 197 | class Client(Protocol): 198 | _message = Accumulate(b"") 199 | 200 | def dataReceived(self, data): 201 | self._message.some_results(self.factory.reactor, data) 202 | 203 | async def next_message(self, expected_size): 204 | return await self._message.next_item(self.factory.reactor, expected_size) 205 | 206 | def send(self, data): 207 | self.transport.write(data) 208 | 209 | 210 | class ServerFactory(Factory): 211 | protocol = Server 212 | noisy = True 213 | 214 | def __init__(self, reactor): 215 | self.reactor = reactor 216 | self._got_protocol = Next() 217 | 218 | async def next_client(self): 219 | return await self._got_protocol.next_item() 220 | 221 | def buildProtocol(self, *args): 222 | p = super().buildProtocol(*args) 223 | self._got_protocol.trigger(self.reactor, p) 224 | return p 225 | 226 | 227 | class ClientFactory(Factory): 228 | protocol = Client 229 | 230 | def __init__(self, reactor): 231 | self.reactor = reactor 232 | -------------------------------------------------------------------------------- /src/fowl/visual.py: -------------------------------------------------------------------------------- 1 | from rich.table import Table 2 | from rich.text import Text 3 | 4 | import time 5 | import random 6 | 7 | import humanize 8 | 9 | from wormhole._status import Connecting 10 | 11 | from .status import FowlStatus 12 | from fowl import chicken 13 | 14 | 15 | def render_status(st: FowlStatus) -> Table: # Panel? seomthing else 16 | """ 17 | Render the given fowl status to a Rich thing 18 | """ 19 | t = Table(show_header=False, show_lines=True) #title="Active Connections") 20 | t.add_column(justify="left", width=8) 21 | t.add_column(justify="left", width=40) 22 | t.add_column(justify="left", width=8) 23 | 24 | status_local = Text(chicken.default[0]) 25 | status_remote = Text(chicken.peer[0]) 26 | message_text = Text("") 27 | t.add_row(status_local, message_text, status_remote) 28 | 29 | if st.url is None: 30 | status_local.stylize("rgb(100,255,0) on rgb(255,0,0)") 31 | status_remote.stylize("rgb(255,255,255) on rgb(255,0,0)") 32 | message_text.append("connecting...") 33 | else: 34 | message_text.append(st.url) 35 | message_text.stylize("green") 36 | message_text.append(f"\nwelcome={st.welcome}\n") 37 | status_local.plain = chicken.default[1] 38 | status_local.stylize("rgb(0,100,100) on rgb(255,255,100)") 39 | 40 | if st.code is not None: 41 | # only display code until we're connected 42 | if st.verifier is None: 43 | message_text.append(Text(f"code: {st.code}\n", "bold")) 44 | 45 | if st.verifier is not None: 46 | nice_verifier = " ".join( 47 | st.verifier[a:a+4] 48 | for a in range(0, len(st.verifier), 4) 49 | ) 50 | message_text.append(nice_verifier) 51 | status_local.plain = chicken.default[2] 52 | status_local.stylize("rgb(0,100,0) on rgb(100,255,100)") 53 | status_remote.plain = chicken.peer[2] 54 | status_remote.stylize("rgb(0,100,0) on rgb(100,255,100)") 55 | 56 | if isinstance(st.mailbox_connection, Connecting): 57 | status_local.stylize("rgb(0,100,100) on rgb(100,255,200)") 58 | elif isinstance(st.mailbox_connection, Connecting): 59 | status_local.stylize("rgb(0,100,100) on rgb(100,255,255)") 60 | 61 | # turn purple if we / they are closing 62 | if st.peer_closing: 63 | status_remote.stylize("rgb(0,100,100) on rgb(255,100,255)") 64 | if st.we_closing: 65 | status_local.stylize("rgb(0,100,100) on rgb(255,100,255)") 66 | 67 | if random.choice("abcdefgh") == "a": 68 | status_local.plain = random.choice(chicken.default) 69 | 70 | for id_, data in st.listeners.items(): 71 | if data.remote: 72 | t.add_row( 73 | Text(""), 74 | Text("{} <--".format(data.connect.split(":")[2]), justify="right"), 75 | Text("{} 🧙".format(data.listen.split(":")[1])), 76 | ) 77 | else: 78 | t.add_row( 79 | Text("🧙 {}".format(data.listen.split(":")[1])), 80 | Text("--> {}".format(data.connect.split(":")[2])), 81 | Text(""), 82 | ) 83 | 84 | for id_, data in st.subchannels.items(): 85 | if data.listener_id in st.listeners: 86 | local = Text(st.listeners[data.listener_id].listen.split(":")[1] + "\nlisten") 87 | remote = Text("connect\n" + str(data.endpoint.split(":")[-1])) 88 | else: 89 | remote = Text("remote\npeer 🧙") 90 | local = Text("connect\n" + str(data.endpoint.split(":")[-1])) 91 | bw = render_bw(data) 92 | t.add_row(local, bw, remote) 93 | 94 | return t 95 | 96 | 97 | interval = 0.25 98 | 99 | 100 | def render_bw(sub): 101 | start = time.time() # FIXME time provuder 102 | if sub.i: 103 | accum = 0 104 | idx = 0 105 | next_time = start - interval 106 | points = [] 107 | for _ in range(25): 108 | while idx < len(sub.i) and sub.i[idx][1] > next_time: 109 | accum += sub.i[idx][0] 110 | idx += 1 111 | 112 | points.append(accum) 113 | accum = 0 114 | next_time = next_time - interval 115 | 116 | bw = "" 117 | for p in points: 118 | if p < 1: 119 | bw += "\u2581" 120 | elif p < 100: 121 | bw += "\u2582" 122 | elif p < 1000: 123 | bw += "\u2583" 124 | elif p < 10000: 125 | bw += "\u2584" 126 | elif p < 100000: 127 | bw += "\u2585" 128 | elif p < 10000000: 129 | bw += "\u2586" 130 | elif p < 10000000000: 131 | bw += "\u2587" 132 | else: 133 | bw += "\u2588" 134 | bw += " " + humanize.naturalsize(sum(x[0] for x in sub.i)) 135 | else: 136 | bw = "" 137 | rendered = Text(bw, style="blue", justify="center") 138 | rendered.append_text(Text("\n" + render_bw_out(sub), style="yellow")) 139 | return rendered 140 | 141 | 142 | def render_bw_out(sub): 143 | start = time.time() 144 | if not sub.o: 145 | return "" 146 | accum = 0 147 | idx = 0 148 | next_time = start - interval 149 | points = [] 150 | for _ in range(25): 151 | while idx < len(sub.o) and sub.o[idx][1] > next_time: 152 | accum += sub.o[idx][0] 153 | idx += 1 154 | 155 | points.append(accum) 156 | accum = 0 157 | next_time = next_time - interval 158 | 159 | bw = humanize.naturalsize(sum(x[0] for x in sub.o)) + " " 160 | for p in reversed(points): 161 | if p < 1: 162 | bw += "\u2581" 163 | elif p < 100: 164 | bw += "\u2582" 165 | elif p < 1000: 166 | bw += "\u2583" 167 | elif p < 10_000: 168 | bw += "\u2584" 169 | elif p < 100_000: 170 | bw += "\u2585" 171 | elif p < 1_000_000: 172 | bw += "\u2586" 173 | elif p < 100_000_000: 174 | bw += "\u2587" 175 | else: 176 | bw += "\u2588" 177 | return bw 178 | -------------------------------------------------------------------------------- /testcase.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import json 4 | from collections import defaultdict 5 | 6 | from twisted.internet.defer import Deferred 7 | from twisted.internet.task import react 8 | from twisted.internet.protocol import ProcessProtocol, Protocol, Factory 9 | from twisted.internet.endpoints import serverFromString, clientFromString 10 | 11 | from fowl.observer import Next, Accumulate 12 | 13 | 14 | class _FowlProtocol(ProcessProtocol): 15 | """ 16 | This speaks to an underlying ``fow`` sub-process. 17 | ``fow`` consumes and emits a line-oriented JSON protocol. 18 | """ 19 | 20 | def __init__(self): 21 | # all messages we've received that _haven't_ yet been asked 22 | # for via next_message() 23 | self._messages = [] 24 | # maps str -> list[Deferred]: kind-string to awaiters 25 | self._message_awaits = defaultdict(list) 26 | self.exited = Deferred() 27 | 28 | def processEnded(self, reason): 29 | self.exited.callback(None) 30 | 31 | def childDataReceived(self, childFD, data): 32 | try: 33 | js = json.loads(data) 34 | except Exception: 35 | print("BAD", data.decode("utf8")) 36 | else: 37 | self._maybe_notify(js) 38 | 39 | def _maybe_notify(self, js): 40 | kind = js["kind"] 41 | if kind in self._message_awaits: 42 | notify, self._message_awaits[kind] = self._message_awaits[kind], list() 43 | for d in notify: 44 | d.callback(js) 45 | else: 46 | self._messages.append(js) 47 | print(js) 48 | 49 | def send_message(self, msg): 50 | data = json.dumps(msg) 51 | self.transport.write(data.encode("utf8") + b"\n") 52 | 53 | def next_message(self, kind): 54 | d = Deferred() 55 | for idx, msg in enumerate(self._messages): 56 | if kind == msg["kind"]: 57 | del self._messages[idx] 58 | d.callback(msg) 59 | return d 60 | self._message_awaits[kind].append(d) 61 | return d 62 | 63 | def all_messages(self, kind=None): 64 | # we _do_ want to make a copy of the list every time 65 | # (so the caller can't "accidentally" mess with our state) 66 | return [ 67 | msg 68 | for msg in self._messages 69 | if kind is None or msg["kind"] == kind 70 | ] 71 | 72 | 73 | @react 74 | async def main(reactor): 75 | host_proto = _FowlProtocol() 76 | reactor.spawnProcess( 77 | host_proto, 78 | sys.executable, 79 | [sys.executable, "-m", "fowl", "--mailbox", "ws://127.0.0.1:4000/v1"], 80 | env={"PYTHONUNBUFFERED": "1"}, 81 | ) 82 | host_proto.send_message({"kind": "allocate-code"}) 83 | msg = await host_proto.next_message("code-allocated") 84 | 85 | guest_proto = _FowlProtocol() 86 | reactor.spawnProcess( 87 | guest_proto, 88 | sys.executable, 89 | [sys.executable, "-m", "fowl", "--mailbox", "ws://127.0.0.1:4000/v1"], 90 | env={"PYTHONUNBUFFERED": "1"}, 91 | ) 92 | guest_proto.send_message({"kind": "set-code", "code": msg["code"]}) 93 | msg = await guest_proto.next_message("code-allocated") 94 | 95 | print("waiting for peers") 96 | await host_proto.next_message("peer-connected") 97 | print("one peer") 98 | await guest_proto.next_message("peer-connected") 99 | print("two peers") 100 | 101 | if 'remote' in sys.argv: 102 | guest_proto.send_message({ 103 | "kind": "grant-permission", 104 | "connect": [1111], 105 | "listen": [], ## this is annoying, fix in fowld 106 | }) 107 | host_proto.send_message({ 108 | "kind": "local", 109 | "listen": "tcp:8888:interface=127.0.0.1", 110 | "connect": "tcp:127.0.0.1:1111", 111 | }) 112 | 113 | m = await host_proto.next_message("listening") 114 | else: 115 | print("local forward") 116 | guest_proto.send_message({ 117 | "kind": "grant-permission", 118 | "connect": [], # FIXME annoying 119 | "listen": [8888], 120 | }) 121 | host_proto.send_message({ 122 | "kind": "grant-permission", 123 | "connect": [1111], 124 | "listen": [], ## this is annoying, fix in fowld 125 | }) 126 | host_proto.send_message({ 127 | "kind": "remote", 128 | "listen": "tcp:8888:interface=127.0.0.1", 129 | "connect": "tcp:127.0.0.1:1111" 130 | }) 131 | m = await guest_proto.next_message("listening") 132 | print("got it", m) 133 | 134 | 135 | class Server(Protocol): 136 | _message = Accumulate(b"") 137 | 138 | def dataReceived(self, data): 139 | self._message.some_results(reactor, data) 140 | 141 | async def next_message(self, expected_size): 142 | return await self._message.next_item(reactor, expected_size) 143 | 144 | def send(self, data): 145 | self.transport.write(data) 146 | 147 | 148 | class Client(Protocol): 149 | _message = Accumulate(b"") 150 | 151 | def dataReceived(self, data): 152 | self._message.some_results(reactor, data) 153 | 154 | async def next_message(self, expected_size): 155 | return await self._message.next_item(reactor, expected_size) 156 | 157 | def send(self, data): 158 | self.transport.write(data) 159 | 160 | 161 | class ServerFactory(Factory): 162 | protocol = Server 163 | noisy = True 164 | _got_protocol = Next() 165 | 166 | async def next_client(self): 167 | return await self._got_protocol.next_item() 168 | 169 | def buildProtocol(self, *args): 170 | p = super().buildProtocol(*args) 171 | self._got_protocol.trigger(reactor, p) 172 | return p 173 | 174 | listener = ServerFactory() 175 | server_port = await serverFromString(reactor, "tcp:1111:interface=127.0.0.1").listen(listener) 176 | 177 | # if we do 'too many' test-cases debian complains about 178 | # "twisted.internet.error.ConnectBindError: Couldn't bind: 24: Too 179 | # many open files." 180 | # gc.collect() doesn't fix it. 181 | who = True 182 | for size in range(2**6, 2**18, 2**10): 183 | print("TEST", size, who) 184 | client = clientFromString(reactor, "tcp:127.0.0.1:8888") 185 | client_proto = await client.connect(Factory.forProtocol(Client)) 186 | server = await listener.next_client() 187 | 188 | data = os.urandom(size) 189 | if who: 190 | client_proto.send(data) 191 | msg = await server.next_message(len(data)) 192 | else: 193 | server.send(data) 194 | msg = await client_proto.next_message(len(data)) 195 | who = not who 196 | assert msg == data, "Incorrect data transfer" 197 | 198 | print(host_proto.all_messages()) 199 | print(guest_proto.all_messages()) 200 | 201 | guest_proto.transport.signalProcess('TERM') 202 | host_proto.transport.signalProcess('TERM') 203 | -------------------------------------------------------------------------------- /update-version.py: -------------------------------------------------------------------------------- 1 | # 2 | # this updates the (tagged) version of the software 3 | # 4 | # we use YY.MM. so the process to update the version is to 5 | # take today's date, start with counter at 0 and increment counter 6 | # until we _don't_ find a tag like that. 7 | # 8 | # e.g. v22.1.0 is the first release in January, 2022 and v22.1.1 is 9 | # the second release in January, 2022, etc. 10 | # 11 | # Any "options" are hard-coded in here (e.g. the GnuPG key to use) 12 | # 13 | 14 | author = "meejah " 15 | 16 | 17 | import sys 18 | import time 19 | import itertools 20 | from datetime import datetime 21 | 22 | from dulwich.repo import Repo 23 | from dulwich.porcelain import ( 24 | tag_list, 25 | tag_create, 26 | status, 27 | ) 28 | 29 | from twisted.internet.task import ( 30 | react, 31 | ) 32 | from twisted.internet.defer import ( 33 | ensureDeferred, 34 | ) 35 | 36 | 37 | def existing_tags(git): 38 | versions = list(v.decode("utf8") for v in tag_list(git)) 39 | return versions 40 | 41 | 42 | def create_new_version(git): 43 | now = datetime.now() 44 | versions = existing_tags(git) 45 | 46 | for counter in itertools.count(): 47 | version = "{}.{}.{}".format(now.year - 2000, now.month, counter) 48 | if version not in versions: 49 | return version 50 | 51 | 52 | async def main(reactor): 53 | git = Repo(".") 54 | 55 | # including untracked files can be very slow (if there are lots, 56 | # like in virtualenvs) and we don't care anyway 57 | st = status(git, untracked_files="no") 58 | if any(st.staged.values()) or st.unstaged: 59 | print("unclean checkout; aborting") 60 | raise SystemExit(1) 61 | 62 | v = create_new_version(git) 63 | if "--no-tag" in sys.argv: 64 | print(v) 65 | return 66 | 67 | print("Existing tags: {}".format(" ".join(existing_tags(git)))) 68 | print("New tag will be {}".format(v)) 69 | 70 | # the "tag time" is seconds from the epoch .. we quantize these to 71 | # the start of the day in question, in UTC. 72 | now = datetime.now() 73 | s = now.utctimetuple() 74 | ts = int( 75 | time.mktime( 76 | time.struct_time((s.tm_year, s.tm_mon, s.tm_mday, 0, 0, 0, 0, s.tm_yday, 0)) 77 | ) 78 | ) 79 | tag_create( 80 | repo=git, 81 | tag=v.encode("utf8"), 82 | author=author.encode("utf8"), 83 | message="Release {}".format(v).encode("utf8"), 84 | annotated=True, 85 | objectish=b"HEAD", 86 | sign=author.encode("utf8"), 87 | tag_time=ts, 88 | tag_timezone=0, 89 | ) 90 | 91 | print("Tag created locally, it is not pushed") 92 | print("To push it run something like:") 93 | print(" git push origin {}".format(v)) 94 | 95 | 96 | if __name__ == "__main__": 97 | react(lambda r: ensureDeferred(main(r))) 98 | --------------------------------------------------------------------------------