├── .flake8 ├── .gitignore ├── LICENSE.txt ├── README.md ├── agent ├── __init__.py ├── main.py ├── tandem │ ├── __init__.py │ ├── agent │ │ ├── __init__.py │ │ ├── configuration.py │ │ ├── executables │ │ │ ├── __init__.py │ │ │ └── agent.py │ │ ├── io │ │ │ ├── __init__.py │ │ │ ├── document.py │ │ │ ├── proxies │ │ │ │ └── relay.py │ │ │ └── std_streams.py │ │ ├── models │ │ │ ├── __init__.py │ │ │ ├── connection.py │ │ │ └── connection_state.py │ │ ├── protocol │ │ │ ├── __init__.py │ │ │ ├── handlers │ │ │ │ ├── __init__.py │ │ │ │ ├── editor.py │ │ │ │ ├── interagent.py │ │ │ │ └── rendezvous.py │ │ │ └── messages │ │ │ │ ├── __init__.py │ │ │ │ ├── editor.py │ │ │ │ └── interagent.py │ │ ├── stores │ │ │ ├── __init__.py │ │ │ └── connection.py │ │ └── utils │ │ │ ├── __init__.py │ │ │ └── hole_punching.py │ └── shared └── test_client.py ├── crdt ├── .eslintignore ├── .eslintrc ├── api │ ├── apply_operations.js │ ├── get_document_operations.js │ ├── get_document_text.js │ ├── index.js │ ├── replicate_document.js │ ├── set_document.js │ └── set_text_in_range.js ├── index.js ├── io │ └── index.js ├── package-lock.json ├── package.json ├── stores │ └── document.js ├── utils │ └── logger.js └── webpack.config.js ├── docs ├── CNAME ├── img │ ├── neovim.png │ ├── preview.png │ ├── sublime.png │ └── vim.png └── index.html ├── lib-python ├── __init__.py ├── io │ ├── __init__.py │ ├── base.py │ ├── proxies │ │ ├── __init__.py │ │ ├── base.py │ │ ├── fragment.py │ │ ├── list_parameters.py │ │ ├── reliability.py │ │ └── unicode.py │ └── udp_gateway.py ├── models │ ├── __init__.py │ ├── base.py │ ├── fragment.py │ └── peer.py ├── protocol │ ├── __init__.py │ ├── handlers │ │ ├── __init__.py │ │ ├── addressed.py │ │ ├── base.py │ │ └── multi.py │ └── messages │ │ ├── __init__.py │ │ ├── base.py │ │ └── rendezvous.py ├── stores │ ├── __init__.py │ ├── base.py │ ├── fragment.py │ └── reliability.py └── utils │ ├── __init__.py │ ├── fragment.py │ ├── proxy.py │ ├── relay.py │ ├── reliability.py │ ├── static_value.py │ └── time_scheduler.py ├── plugins ├── sublime │ ├── Main.sublime-menu │ ├── README.md │ ├── __init__.py │ ├── diff_match_patch.py │ ├── edit.py │ ├── enum-dist │ │ ├── MANIFEST.in │ │ ├── PKG-INFO │ │ ├── README │ │ ├── enum │ │ │ ├── LICENSE │ │ │ ├── README │ │ │ ├── __init__.py │ │ │ ├── doc │ │ │ │ ├── enum.pdf │ │ │ │ └── enum.rst │ │ │ └── test.py │ │ ├── enum34.egg-info │ │ │ ├── PKG-INFO │ │ │ ├── SOURCES.txt │ │ │ ├── dependency_links.txt │ │ │ └── top_level.txt │ │ ├── setup.cfg │ │ └── setup.py │ ├── sublime_dev_setup.sh │ ├── tandem.py │ └── tandem.sublime-commands └── vim │ ├── README_nvim.md │ ├── README_vim.md │ ├── neovim_dev_setup.sh │ ├── tandem_lib │ ├── __init__.py │ ├── agent │ ├── diff_match_patch.py │ └── tandem_plugin.py │ ├── tandem_neovim.py │ └── tandem_vim.vim ├── release_scripts ├── README.md ├── prepare_scripts │ ├── nvim.sh │ ├── sublime.sh │ └── vim.sh ├── release.py └── tandem_release.sh └── rendezvous ├── __init__.py ├── main.py ├── prototype ├── client.py └── server.py ├── tandem-rendezvous.conf ├── tandem ├── __init__.py ├── rendezvous │ ├── __init__.py │ ├── executables │ │ ├── __init__.py │ │ └── rendezvous.py │ ├── io │ │ ├── __init__.py │ │ └── proxies │ │ │ └── relay.py │ ├── models │ │ ├── __init__.py │ │ ├── connection.py │ │ └── session.py │ ├── protocol │ │ ├── __init__.py │ │ └── handlers │ │ │ ├── __init__.py │ │ │ └── agent.py │ └── stores │ │ ├── __init__.py │ │ └── session.py └── shared └── test_client.py /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E722 3 | exclude = plugins 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.sw* 3 | *.pyc 4 | node_modules 5 | build 6 | *.log 7 | 8 | .env 9 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Tandem 2 | 3 | Tandem is a decentralized, collaborative text-editing solution. Tandem works 4 | with native text editors, works across different editors, and uses peer-to-peer 5 | connections to facilitate communication. 6 | 7 | Tandem exists as a set of plugins for native text editors. We currently support 8 | Sublime Text 3 and Neovim. We also unofficially support Vim. 9 | 10 | Collaborating is as easy as installing the plugin on your editor and creating a 11 | Tandem Session. Invite other people to your session, and get typing in tandem! 12 | 13 | ## Installation 14 | 15 | ### Requirements 16 | Tandem is currently supported on Mac OS X - it may work on Linux, but use at 17 | your own risk. 18 | To use Tandem, your environment must have `python3.6+` and `node.js` installed 19 | (tested and confirmed working on `node.js v7.7.4`). 20 | `python3.6+` is a requirement for our agent code and `node.js` is required by 21 | our CRDT. 22 | 23 | ### Plugins 24 | Please follow the installation guides for your plugin of choice: 25 | - [Sublime](https://github.com/typeintandem/sublime) 26 | - [Neovim](https://github.com/typeintandem/nvim) 27 | - [Vim (unofficially supported)](https://github.com/typeintandem/vim) 28 | 29 | ## How it Works 30 | Tandem is split into four components: the editor plugins, the networking agent, 31 | the conflict-free replicated data type (CRDT) solution, and the rendezvous 32 | server. 33 | 34 | ### Editor Plugins 35 | The plugins interface with the text buffer in your plugin, detecting local 36 | changes and applying remote changes, and allowing users to create, join and 37 | leave sessions. Each plugin has its own repository and code. 38 | 39 | ### Agent 40 | The agent establishes connections between other peers connected to your 41 | collaborative session. It takes messages from the editor plugin and broadcasts 42 | them to all peers, and with the help of the CRDT instructs the editor to apply 43 | remote changes to your local text buffer. 44 | 45 | ### CRDT 46 | The CRDT is used to represent the state of your local document, and transforms 47 | document edits into operations that can be applied remotely, without conflicts. 48 | We submit these conflict-free operations to other peers via the agent. 49 | 50 | *Instead of reinventing the wheel and writing the CRDT ourselves, we leveraged 51 | the work done by GitHub's team working on Teletype. We used [their 52 | CRDT](https://github.com/atom/teletype-crdt) under the hood and instead 53 | focussed our efforts on integrating the CRDT with different editors.* 54 | 55 | ### Rendezvous Server 56 | The rendezvous server is used to help establish peer-to-peer connections. It 57 | server notes the connection details of any peer that joins a session. When any 58 | subsequent peer wants to join, the server provides the connection details of 59 | all other peers in the session so that they can communicate directly with each 60 | other. 61 | 62 | **Note: Peer-to-peer connections cannot always be established. As a convenience 63 | to the user, the rendezvous server will also act as a relay to send data 64 | between agents when this happens.** 65 | If you wish to disable this behaviour, you can do so by disabling the 66 | `USE_RELAY` flag in: `agent/tandem/agent/configuration.py` 67 | 68 | ## Terms of Service 69 | By using Tandem, you agree that any modified versions of Tandem will not use 70 | the rendezvous server hosted by the owners. You must host and use your own copy 71 | of the rendezvous server. We want to provide a good user experience for Tandem, 72 | and it would be difficult to do that with modified clients as well. 73 | 74 | You can launch the rendezvous server by running `python3 ./rendezvous/main.py`. 75 | Change the address of the rendezvous server used by the agent in the 76 | configuration file to point to your server's host. This file is located at: 77 | `agent/tandem/agent/configuration.py` 78 | 79 | ## License 80 | Copyright (c) 2018 Team Lightly 81 | 82 | See [LICENSE.txt](LICENSE.txt) 83 | 84 | Licensed under the Apache License, Version 2.0 (the "License"); 85 | you may not use this file except in compliance with the License. 86 | You may obtain a copy of the License at: 87 | 88 | http://www.apache.org/licenses/LICENSE-2.0 89 | 90 | ## Authors 91 | Team Lightly 92 | [Geoffrey Yu](https://github.com/geoffxy), [Jamiboy 93 | Mohammad](https://github.com/jamiboym) and [Sameer 94 | Chitley](https://github.com/rageandqq) 95 | 96 | We are a team of senior Software Engineering students at the University of 97 | Waterloo. 98 | Tandem was created as our [Engineering Capstone Design 99 | Project](https://uwaterloo.ca/capstone-design). 100 | -------------------------------------------------------------------------------- /agent/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/agent/__init__.py -------------------------------------------------------------------------------- /agent/main.py: -------------------------------------------------------------------------------- 1 | import signal 2 | import logging 3 | import threading 4 | import argparse 5 | from tandem.agent.executables.agent import TandemAgent 6 | 7 | should_shutdown = threading.Event() 8 | 9 | 10 | def signal_handler(signal, frame): 11 | global should_shutdown 12 | should_shutdown.set() 13 | 14 | 15 | def set_up_logging(log_location): 16 | logging.basicConfig( 17 | level=logging.DEBUG, 18 | format="%(asctime)s %(levelname)-8s %(message)s", 19 | datefmt="%Y-%m-%d %H:%M", 20 | filename=log_location, 21 | filemode="w", 22 | ) 23 | 24 | 25 | def main(): 26 | signal.signal(signal.SIGINT, signal_handler) 27 | signal.signal(signal.SIGTERM, signal_handler) 28 | 29 | parser = argparse.ArgumentParser(description="Starts the Tandem agent.") 30 | parser.add_argument( 31 | "--host", 32 | default="", 33 | help="The host address to bind to.", 34 | ) 35 | parser.add_argument( 36 | "--port", 37 | default=0, 38 | type=int, 39 | help="The port to listen on.", 40 | ) 41 | parser.add_argument( 42 | "--log-file", 43 | default="/tmp/tandem-agent.log", 44 | help="The location of the log file.", 45 | ) 46 | args = parser.parse_args() 47 | 48 | set_up_logging(args.log_file) 49 | 50 | # Run the agent until asked to terminate 51 | with TandemAgent(args.host, args.port): 52 | should_shutdown.wait() 53 | 54 | 55 | if __name__ == "__main__": 56 | main() 57 | -------------------------------------------------------------------------------- /agent/tandem/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/agent/tandem/__init__.py -------------------------------------------------------------------------------- /agent/tandem/agent/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/agent/tandem/agent/__init__.py -------------------------------------------------------------------------------- /agent/tandem/agent/configuration.py: -------------------------------------------------------------------------------- 1 | import os 2 | import socket 3 | 4 | # Tandem will try to establish a direct connection with other peers in a 5 | # session. However, this is not always possible. When Tandem is unable to 6 | # establish a peer-to-peer connection, we will relay messages to each peer 7 | # through our servers. If this is undesirable for your use case, you can set 8 | # this flag to "False". 9 | # 10 | # Please note that with relay disabled, you will not be able to collaborate 11 | # with any peers that Tandem cannot reach directly. Tandem does not notify you 12 | # if a peer-to-peer connection cannot be established. 13 | USE_RELAY = True 14 | 15 | # DO NOT edit anything below this unless you know what you're doing! 16 | 17 | PROJECT_ROOT = os.path.join( 18 | os.path.dirname(os.path.abspath(__file__)), 19 | '..', 20 | '..', 21 | '..', 22 | ) 23 | BASE_DIR = os.path.dirname(PROJECT_ROOT) 24 | CRDT_PATH = os.path.join(BASE_DIR, "..", "crdt") 25 | PLUGIN_PATH = os.path.join(BASE_DIR, "..", "plugins") 26 | RENDEZVOUS_ADDRESS = ( 27 | socket.gethostbyname("rendezvous.typeintandem.com"), 28 | 60000, 29 | ) 30 | -------------------------------------------------------------------------------- /agent/tandem/agent/executables/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/agent/tandem/agent/executables/__init__.py -------------------------------------------------------------------------------- /agent/tandem/agent/executables/agent.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import uuid 3 | from tandem.agent.io.document import Document 4 | from tandem.agent.io.std_streams import STDStreams 5 | from tandem.shared.io.udp_gateway import UDPGateway 6 | from tandem.agent.protocol.handlers.editor import EditorProtocolHandler 7 | from tandem.agent.protocol.handlers.interagent import InteragentProtocolHandler 8 | from tandem.agent.protocol.handlers.rendezvous import RendezvousProtocolHandler 9 | from tandem.shared.protocol.handlers.multi import MultiProtocolHandler 10 | from tandem.shared.utils.time_scheduler import TimeScheduler 11 | from tandem.shared.io.proxies.fragment import FragmentProxy 12 | from tandem.shared.io.proxies.list_parameters import ListParametersProxy 13 | from tandem.shared.io.proxies.unicode import UnicodeProxy 14 | from tandem.shared.io.proxies.reliability import ReliabilityProxy 15 | from tandem.agent.io.proxies.relay import AgentRelayProxy 16 | from concurrent.futures import ThreadPoolExecutor 17 | from tandem.agent.configuration import RENDEZVOUS_ADDRESS 18 | 19 | 20 | class TandemAgent: 21 | def __init__(self, host, port): 22 | self._id = uuid.uuid4() 23 | self._requested_host = host 24 | # This is the port the user specified on the command line (it can be 0) 25 | self._requested_port = port 26 | self._main_executor = ThreadPoolExecutor(max_workers=1) 27 | self._time_scheduler = TimeScheduler(self._main_executor) 28 | self._document = Document() 29 | self._std_streams = STDStreams(self._on_std_input) 30 | self._interagent_gateway = UDPGateway( 31 | self._requested_host, 32 | self._requested_port, 33 | self._gateway_message_handler, 34 | [ 35 | ListParametersProxy(), 36 | UnicodeProxy(), 37 | FragmentProxy(), 38 | AgentRelayProxy(RENDEZVOUS_ADDRESS), 39 | ReliabilityProxy(self._time_scheduler), 40 | ], 41 | ) 42 | self._editor_protocol = EditorProtocolHandler( 43 | self._id, 44 | self._std_streams, 45 | self._interagent_gateway, 46 | self._document, 47 | ) 48 | self._interagent_protocol = InteragentProtocolHandler( 49 | self._id, 50 | self._std_streams, 51 | self._interagent_gateway, 52 | self._document, 53 | self._time_scheduler, 54 | ) 55 | self._rendezvous_protocol = RendezvousProtocolHandler( 56 | self._id, 57 | self._interagent_gateway, 58 | self._time_scheduler, 59 | self._document, 60 | ) 61 | self._gateway_handlers = MultiProtocolHandler( 62 | self._interagent_protocol, 63 | self._rendezvous_protocol, 64 | ) 65 | 66 | def __enter__(self): 67 | self.start() 68 | return self 69 | 70 | def __exit__(self, exc_type, exc_value, traceback): 71 | self.stop() 72 | 73 | def start(self): 74 | self._time_scheduler.start() 75 | self._document.start() 76 | self._std_streams.start() 77 | self._interagent_gateway.start() 78 | logging.info("Tandem Agent has started.") 79 | 80 | def stop(self): 81 | def atomic_shutdown(): 82 | self._interagent_protocol.stop() 83 | self._interagent_gateway.stop() 84 | self._std_streams.stop() 85 | self._document.stop() 86 | self._time_scheduler.stop() 87 | self._main_executor.submit(atomic_shutdown) 88 | self._main_executor.shutdown() 89 | logging.info("Tandem Agent has shut down.") 90 | 91 | def _on_std_input(self, retrieve_data): 92 | # Called by _std_streams after receiving a new message from the plugin 93 | self._main_executor.submit( 94 | self._editor_protocol.handle_message, 95 | retrieve_data, 96 | ) 97 | 98 | def _gateway_message_handler(self, retrieve_data): 99 | # Do not call directly - called by _interagent_gateway 100 | self._main_executor.submit( 101 | self._gateway_handlers.handle_raw_data, 102 | retrieve_data, 103 | ) 104 | -------------------------------------------------------------------------------- /agent/tandem/agent/io/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/agent/tandem/agent/io/__init__.py -------------------------------------------------------------------------------- /agent/tandem/agent/io/document.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | from subprocess import Popen, PIPE 4 | from tandem.agent.configuration import CRDT_PATH 5 | 6 | CRDT_PROCESS = ["node", os.path.join(CRDT_PATH, "build", "bundle.js")] 7 | 8 | 9 | class Document: 10 | def __init__(self): 11 | self._crdt_process = None 12 | self._pending_remote_operations = [] 13 | self._write_request_sent = False 14 | 15 | def start(self): 16 | self._crdt_process = Popen( 17 | CRDT_PROCESS, 18 | stdin=PIPE, 19 | stdout=PIPE, 20 | encoding="utf-8", 21 | ) 22 | 23 | def stop(self): 24 | self._crdt_process.stdin.close() 25 | self._crdt_process.terminate() 26 | self._crdt_process.wait() 27 | 28 | def apply_operations(self, operations_list): 29 | return self._call_remote_function( 30 | "applyOperations", 31 | [operations_list], 32 | ) 33 | 34 | def get_document_text(self): 35 | return self._call_remote_function("getDocumentText") 36 | 37 | def set_text_in_range(self, start, end, text): 38 | return self._call_remote_function( 39 | "setTextInRange", 40 | [start, end, text], 41 | ) 42 | 43 | def get_document_operations(self): 44 | return self._call_remote_function("getDocumentOperations") 45 | 46 | def enqueue_remote_operations(self, operations_list): 47 | self._pending_remote_operations.extend(operations_list) 48 | 49 | def apply_queued_operations(self): 50 | text_patches = self.apply_operations(self._pending_remote_operations) 51 | self._pending_remote_operations.clear() 52 | return text_patches 53 | 54 | def write_request_sent(self): 55 | return self._write_request_sent 56 | 57 | def set_write_request_sent(self, value): 58 | self._write_request_sent = value 59 | 60 | def _call_remote_function(self, function_name, parameters=None): 61 | call_message = {"function": function_name} 62 | if parameters is not None: 63 | call_message["parameters"] = parameters 64 | self._crdt_process.stdin.write(json.dumps(call_message)) 65 | self._crdt_process.stdin.write("\n") 66 | self._crdt_process.stdin.flush() 67 | 68 | response = json.loads(self._crdt_process.stdout.readline()) 69 | return response["value"] 70 | -------------------------------------------------------------------------------- /agent/tandem/agent/io/proxies/relay.py: -------------------------------------------------------------------------------- 1 | from tandem.shared.io.proxies.base import ProxyBase 2 | from tandem.shared.utils.relay import RelayUtils 3 | from tandem.shared.io.udp_gateway import UDPGateway 4 | from tandem.agent.stores.connection import ConnectionStore 5 | 6 | 7 | class AgentRelayProxy(ProxyBase): 8 | def __init__(self, relay_server_address): 9 | self._relay_server_address = relay_server_address 10 | 11 | def should_relay(self, address): 12 | connection_store = ConnectionStore.get_instance() 13 | connection = connection_store.get_connection_by_address(address) 14 | return ( 15 | self._relay_server_address != address and 16 | connection and connection.is_relayed() 17 | ) 18 | 19 | def pre_write_io_data(self, params): 20 | args, kwargs = params 21 | io_datas, = args 22 | 23 | new_io_datas = [] 24 | for io_data in io_datas: 25 | new_io_data = io_data 26 | if self.should_relay(io_data.get_address()): 27 | new_raw_data = RelayUtils.serialize( 28 | io_data.get_data(), 29 | io_data.get_address(), 30 | ) 31 | new_io_data = UDPGateway.data_class( 32 | new_raw_data, 33 | self._relay_server_address, 34 | ) 35 | new_io_datas.append(new_io_data) 36 | 37 | new_args = (new_io_datas,) 38 | return (new_args, kwargs) 39 | 40 | def on_retrieve_io_data(self, params): 41 | args, kwargs = params 42 | if args is None or args is (None, None): 43 | return params 44 | 45 | raw_data, address = args 46 | 47 | if RelayUtils.is_relay(raw_data): 48 | new_data, new_address = RelayUtils.deserialize(raw_data) 49 | new_args = new_data, new_address 50 | return (new_args, kwargs) 51 | else: 52 | return params 53 | -------------------------------------------------------------------------------- /agent/tandem/agent/io/std_streams.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import logging 3 | from tandem.shared.io.base import InterfaceDataBase, InterfaceBase 4 | 5 | 6 | class STDData(InterfaceDataBase): 7 | pass 8 | 9 | 10 | class STDStreams(InterfaceBase): 11 | data_class = STDData 12 | 13 | def __init__(self, handler_function): 14 | super(STDStreams, self).__init__(handler_function) 15 | 16 | def stop(self): 17 | super(STDStreams, self).stop() 18 | sys.stdout.close() 19 | 20 | def write_io_data(self, *args, **kwargs): 21 | io_data, = args 22 | 23 | sys.stdout.write(io_data.get_data()) 24 | sys.stdout.write("\n") 25 | sys.stdout.flush() 26 | 27 | def _read_data(self): 28 | try: 29 | for line in sys.stdin: 30 | self._received_data(line) 31 | except: 32 | logging.exception("Exception when reading from stdin:") 33 | raise 34 | -------------------------------------------------------------------------------- /agent/tandem/agent/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/agent/tandem/agent/models/__init__.py -------------------------------------------------------------------------------- /agent/tandem/agent/models/connection.py: -------------------------------------------------------------------------------- 1 | from tandem.shared.models.base import ModelBase 2 | from tandem.agent.models.connection_state import ConnectionState 3 | import logging 4 | 5 | 6 | class Connection(ModelBase): 7 | def __init__(self, peer): 8 | self._peer = peer 9 | 10 | def get_id(self): 11 | return self._peer.get_id() 12 | 13 | def get_active_address(self): 14 | raise NotImplementedError 15 | 16 | def get_connection_state(self): 17 | raise NotImplementedError 18 | 19 | def set_connection_state(self, state): 20 | raise NotImplementedError 21 | 22 | def is_relayed(self): 23 | return self.get_connection_state() == ConnectionState.RELAY 24 | 25 | def get_peer(self): 26 | return self._peer 27 | 28 | 29 | class DirectConnection(Connection): 30 | """ 31 | A connection to a peer established without using hole punching. 32 | """ 33 | def __init__(self, peer): 34 | super(DirectConnection, self).__init__(peer) 35 | 36 | def get_active_address(self): 37 | return self.get_peer().get_public_address() 38 | 39 | def get_connection_state(self): 40 | return ConnectionState.OPEN 41 | 42 | def set_connection_state(self, state): 43 | pass 44 | 45 | 46 | class HolePunchedConnection(Connection): 47 | """ 48 | A connection to a peer that was established with hole punching. 49 | """ 50 | PROMOTE_AFTER = 3 51 | 52 | def __init__(self, peer, initiated_connection): 53 | super(HolePunchedConnection, self).__init__(peer) 54 | self._active_address = None 55 | self._interval_handle = None 56 | self._connection_state = ConnectionState.PING 57 | # If true, this agent initiated the connection to this peer 58 | self._initiated_connection = initiated_connection 59 | 60 | self._address_ping_counts = {} 61 | self._address_ping_counts[peer.get_public_address()] = 0 62 | if peer.get_private_address() is not None: 63 | self._address_ping_counts[peer.get_private_address()] = 0 64 | 65 | def get_active_address(self): 66 | if self._active_address is None: 67 | self._active_address = self._compute_active_address() 68 | return self._active_address 69 | 70 | def get_connection_state(self): 71 | return self._connection_state 72 | 73 | def set_connection_state(self, state): 74 | if self._connection_state == state: 75 | return 76 | self._connection_state = state 77 | if self._interval_handle is not None: 78 | self._interval_handle.cancel() 79 | self._interval_handle = None 80 | 81 | def set_interval_handle(self, interval_handle): 82 | self._interval_handle = interval_handle 83 | 84 | def bump_ping_count(self, address): 85 | if address in self._address_ping_counts: 86 | self._address_ping_counts[address] += 1 87 | 88 | def initiated_connection(self): 89 | return self._initiated_connection 90 | 91 | def _compute_active_address(self): 92 | if self.is_relayed(): 93 | return self.get_peer().get_public_address() 94 | 95 | private_address = self.get_peer().get_private_address() 96 | private_address_count = ( 97 | self._address_ping_counts[private_address] 98 | if private_address is not None else 0 99 | ) 100 | 101 | # If the private address is routable, always choose it 102 | if private_address_count > 0: 103 | return ( 104 | private_address 105 | if private_address_count >= HolePunchedConnection.PROMOTE_AFTER 106 | else None 107 | ) 108 | 109 | public_address = self.get_peer().get_public_address() 110 | public_address_count = self._address_ping_counts[public_address] 111 | return ( 112 | public_address 113 | if public_address_count >= HolePunchedConnection.PROMOTE_AFTER 114 | else None 115 | ) 116 | -------------------------------------------------------------------------------- /agent/tandem/agent/models/connection_state.py: -------------------------------------------------------------------------------- 1 | import enum 2 | 3 | 4 | class ConnectionState(enum.Enum): 5 | PING = "ping" 6 | SEND_SYN = "syn" 7 | WAIT_FOR_SYN = "wait" 8 | OPEN = "open" 9 | RELAY = "relay" 10 | UNREACHABLE = "unreachable" 11 | -------------------------------------------------------------------------------- /agent/tandem/agent/protocol/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/agent/tandem/agent/protocol/__init__.py -------------------------------------------------------------------------------- /agent/tandem/agent/protocol/handlers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/agent/tandem/agent/protocol/handlers/__init__.py -------------------------------------------------------------------------------- /agent/tandem/agent/protocol/handlers/editor.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import logging 4 | import socket 5 | import uuid 6 | import tandem.agent.protocol.messages.editor as em 7 | from tandem.agent.protocol.messages.interagent import ( 8 | InteragentProtocolUtils, 9 | NewOperations, 10 | Hello 11 | ) 12 | from tandem.agent.stores.connection import ConnectionStore 13 | from tandem.shared.protocol.messages.rendezvous import ( 14 | RendezvousProtocolUtils, 15 | ConnectRequest, 16 | ) 17 | from tandem.agent.configuration import RENDEZVOUS_ADDRESS 18 | 19 | 20 | class EditorProtocolHandler: 21 | def __init__(self, id, std_streams, gateway, document): 22 | self._id = id 23 | self._std_streams = std_streams 24 | self._gateway = gateway 25 | self._document = document 26 | 27 | def handle_message(self, retrieve_io_data): 28 | io_data = retrieve_io_data() 29 | data = io_data.get_data() 30 | 31 | try: 32 | message = em.deserialize(data) 33 | if type(message) is em.ConnectTo: 34 | self._handle_connect_to(message) 35 | elif type(message) is em.WriteRequestAck: 36 | self._handle_write_request_ack(message) 37 | elif type(message) is em.NewPatches: 38 | self._handle_new_patches(message) 39 | elif type(message) is em.CheckDocumentSync: 40 | self._handle_check_document_sync(message) 41 | elif type(message) is em.HostSession: 42 | self._handle_host_session(message) 43 | elif type(message) is em.JoinSession: 44 | self._handle_join_session(message) 45 | except em.EditorProtocolMarshalError: 46 | logging.info("Ignoring invalid editor protocol message.") 47 | except: 48 | logging.exception( 49 | "Exception when handling editor protocol message:") 50 | raise 51 | 52 | def _handle_connect_to(self, message): 53 | hostname = socket.gethostbyname(message.host) 54 | logging.info( 55 | "Tandem Agent is attempting to establish a direct" 56 | " connection to {}:{}.".format(hostname, message.port), 57 | ) 58 | 59 | address = (hostname, message.port) 60 | payload = InteragentProtocolUtils.serialize(Hello( 61 | id=str(self._id), 62 | should_reply=True, 63 | )) 64 | io_data = self._gateway.generate_io_data(payload, address) 65 | self._gateway.write_io_data(io_data) 66 | 67 | def _handle_write_request_ack(self, message): 68 | logging.debug("Received ACK for seq: {}".format(message.seq)) 69 | text_patches = self._document.apply_queued_operations() 70 | self._document.set_write_request_sent(False) 71 | # Even if no text patches need to be applied, we need to reply to 72 | # the plugin to allow it to accept changes from the user again 73 | text_patches_message = em.ApplyPatches(text_patches) 74 | io_data = self._std_streams.generate_io_data( 75 | em.serialize(text_patches_message), 76 | ) 77 | self._std_streams.write_io_data(io_data) 78 | logging.debug( 79 | "Sent apply patches message for seq: {}".format(message.seq), 80 | ) 81 | 82 | def _handle_new_patches(self, message): 83 | nested_operations = [ 84 | self._document.set_text_in_range( 85 | patch["start"], 86 | patch["end"], 87 | patch["text"], 88 | ) 89 | for patch in message.patch_list 90 | ] 91 | operations = [] 92 | for operations_list in nested_operations: 93 | operations.extend(operations_list) 94 | 95 | connections = ConnectionStore.get_instance().get_open_connections() 96 | if len(connections) == 0: 97 | return 98 | 99 | addresses = [ 100 | connection.get_active_address() for connection in connections 101 | ] 102 | payload = InteragentProtocolUtils.serialize(NewOperations( 103 | operations_list=json.dumps(operations) 104 | )) 105 | io_data = self._gateway.generate_io_data(payload, addresses) 106 | self._gateway.write_io_data( 107 | io_data, 108 | reliability=True, 109 | ) 110 | 111 | def _handle_check_document_sync(self, message): 112 | document_text_content = self._document.get_document_text() 113 | 114 | # TODO: ignore all other messages until we receive an ack 115 | contents = os.linesep.join(message.contents) + os.linesep 116 | 117 | if (contents != document_text_content): 118 | document_lines = document_text_content.split(os.linesep) 119 | apply_text = em.serialize(em.ApplyText(document_lines)) 120 | io_data = self._std_streams.generate_io_data(apply_text) 121 | self._std_streams.write_io_data(io_data) 122 | 123 | def _handle_host_session(self, message): 124 | # Register with rendezvous 125 | session_id = uuid.uuid4() 126 | self._send_connect_request(session_id) 127 | 128 | # Inform plugin of session id 129 | session_info = em.serialize(em.SessionInfo(session_id=str(session_id))) 130 | io_data = self._std_streams.generate_io_data(session_info) 131 | self._std_streams.write_io_data(io_data) 132 | 133 | def _handle_join_session(self, message): 134 | # Parse ID to make sure it's a UUID 135 | session_id = uuid.UUID(message.session_id) 136 | self._send_connect_request(session_id) 137 | 138 | def _send_connect_request(self, session_id): 139 | io_data = self._gateway.generate_io_data( 140 | RendezvousProtocolUtils.serialize(ConnectRequest( 141 | session_id=str(session_id), 142 | my_id=str(self._id), 143 | private_address=( 144 | socket.gethostbyname(socket.gethostname()), 145 | self._gateway.get_port(), 146 | ), 147 | )), 148 | RENDEZVOUS_ADDRESS, 149 | ) 150 | self._gateway.write_io_data(io_data) 151 | -------------------------------------------------------------------------------- /agent/tandem/agent/protocol/handlers/interagent.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import json 3 | import uuid 4 | import tandem.agent.protocol.messages.editor as em 5 | 6 | from tandem.agent.models.connection import DirectConnection 7 | from tandem.agent.models.connection_state import ConnectionState 8 | from tandem.agent.protocol.messages.interagent import ( 9 | InteragentProtocolMessageType, 10 | InteragentProtocolUtils, 11 | NewOperations, 12 | Bye, 13 | Hello, 14 | PingBack, 15 | ) 16 | from tandem.agent.stores.connection import ConnectionStore 17 | from tandem.agent.utils.hole_punching import HolePunchingUtils 18 | from tandem.shared.models.peer import Peer 19 | from tandem.shared.protocol.handlers.addressed import AddressedHandler 20 | from tandem.shared.utils.static_value import static_value as staticvalue 21 | 22 | 23 | class InteragentProtocolHandler(AddressedHandler): 24 | @staticvalue 25 | def _protocol_message_utils(self): 26 | return InteragentProtocolUtils 27 | 28 | @staticvalue 29 | def _protocol_message_handlers(self): 30 | return { 31 | InteragentProtocolMessageType.Ping.value: self._handle_ping, 32 | InteragentProtocolMessageType.PingBack.value: 33 | self._handle_pingback, 34 | InteragentProtocolMessageType.Syn.value: self._handle_syn, 35 | InteragentProtocolMessageType.Hello.value: self._handle_hello, 36 | InteragentProtocolMessageType.Bye.value: self._handle_bye, 37 | InteragentProtocolMessageType.NewOperations.value: 38 | self._handle_new_operations, 39 | } 40 | 41 | def __init__(self, id, std_streams, gateway, document, time_scheduler): 42 | self._id = id 43 | self._std_streams = std_streams 44 | self._gateway = gateway 45 | self._document = document 46 | self._time_scheduler = time_scheduler 47 | self._next_editor_sequence = 0 48 | 49 | def _handle_ping(self, message, sender_address): 50 | peer_id = uuid.UUID(message.id) 51 | connection = \ 52 | ConnectionStore.get_instance().get_connection_by_id(peer_id) 53 | 54 | # Only reply to peers we know about to prevent the other peer from 55 | # thinking it can reach us successfully 56 | if connection is None: 57 | return 58 | 59 | logging.debug( 60 | "Replying to ping from {} at {}:{}." 61 | .format(message.id, *sender_address), 62 | ) 63 | io_data = self._gateway.generate_io_data( 64 | InteragentProtocolUtils.serialize(PingBack(id=str(self._id))), 65 | sender_address, 66 | ) 67 | self._gateway.write_io_data(io_data) 68 | 69 | def _handle_pingback(self, message, sender_address): 70 | peer_id = uuid.UUID(message.id) 71 | connection = \ 72 | ConnectionStore.get_instance().get_connection_by_id(peer_id) 73 | # Only count PingBack messages from peers we know about and from whom 74 | # we expect PingBack messages 75 | if (connection is None or 76 | connection.get_connection_state() != ConnectionState.PING): 77 | return 78 | 79 | logging.debug( 80 | "Counting ping from {} at {}:{}." 81 | .format(message.id, *sender_address), 82 | ) 83 | connection.bump_ping_count(sender_address) 84 | 85 | # When the connection is ready to transition into the SYN/WAIT states, 86 | # an active address will be available 87 | if connection.get_active_address() is None: 88 | return 89 | 90 | connection.set_connection_state( 91 | ConnectionState.SEND_SYN 92 | if connection.initiated_connection() 93 | else ConnectionState.WAIT_FOR_SYN 94 | ) 95 | logging.debug( 96 | "Promoted peer from {} with address {}:{}." 97 | .format(message.id, *(connection.get_active_address())), 98 | ) 99 | 100 | if connection.get_connection_state() == ConnectionState.SEND_SYN: 101 | logging.debug( 102 | "Will send SYN to {} at {}:{}" 103 | .format(message.id, *(connection.get_active_address())), 104 | ) 105 | connection.set_interval_handle(self._time_scheduler.run_every( 106 | HolePunchingUtils.SYN_INTERVAL, 107 | HolePunchingUtils.generate_send_syn( 108 | self._gateway, 109 | connection.get_active_address(), 110 | ), 111 | )) 112 | else: 113 | logging.debug( 114 | "Will wait for SYN from {} at {}:{}" 115 | .format(message.id, *(connection.get_active_address())), 116 | ) 117 | 118 | def _handle_syn(self, message, sender_address): 119 | logging.debug("Received SYN from {}:{}".format(*sender_address)) 120 | connection = ( 121 | ConnectionStore.get_instance() 122 | .get_connection_by_address(sender_address) 123 | ) 124 | if (connection is None or 125 | connection.get_connection_state() == ConnectionState.SEND_SYN): 126 | return 127 | 128 | connection.set_connection_state(ConnectionState.OPEN) 129 | self._send_all_operations(connection, even_if_empty=True) 130 | logging.debug( 131 | "Connection to peer at {}:{} is open." 132 | .format(*(connection.get_active_address())), 133 | ) 134 | 135 | def _handle_hello(self, message, sender_address): 136 | id = uuid.UUID(message.id) 137 | new_connection = DirectConnection(Peer( 138 | id=id, 139 | public_address=sender_address, 140 | )) 141 | ConnectionStore.get_instance().add_connection(new_connection) 142 | logging.info( 143 | "Tandem Agent established a direct connection to {}:{}" 144 | .format(*sender_address), 145 | ) 146 | 147 | if message.should_reply: 148 | io_data = self._gateway.generate_io_data( 149 | InteragentProtocolUtils.serialize(Hello( 150 | id=str(self._id), 151 | should_reply=False, 152 | )), 153 | sender_address, 154 | ) 155 | self._gateway.write_io_data(io_data) 156 | 157 | self._send_all_operations(new_connection) 158 | 159 | def _handle_bye(self, message, sender_address): 160 | connection_store = ConnectionStore.get_instance() 161 | connection = connection_store.get_connection_by_address(sender_address) 162 | if connection is None: 163 | return 164 | connection_store.remove_connection(connection) 165 | 166 | def _handle_new_operations(self, message, sender_address): 167 | connection = ( 168 | ConnectionStore.get_instance() 169 | .get_connection_by_address(sender_address) 170 | ) 171 | if (connection is not None and 172 | connection.get_connection_state() == ConnectionState.SEND_SYN): 173 | connection.set_connection_state(ConnectionState.OPEN) 174 | logging.debug( 175 | "Connection to peer at {}:{} is open." 176 | .format(*(connection.get_active_address())), 177 | ) 178 | 179 | operations_list = json.loads(message.operations_list) 180 | if len(operations_list) == 0: 181 | return 182 | self._document.enqueue_remote_operations(operations_list) 183 | 184 | if not self._document.write_request_sent(): 185 | io_data = self._std_streams.generate_io_data( 186 | em.serialize(em.WriteRequest(self._next_editor_sequence)), 187 | ) 188 | self._std_streams.write_io_data(io_data) 189 | self._document.set_write_request_sent(True) 190 | logging.debug( 191 | "Sent write request seq: {}" 192 | .format(self._next_editor_sequence), 193 | ) 194 | self._next_editor_sequence += 1 195 | 196 | def _send_all_operations(self, connection, even_if_empty=False): 197 | operations = self._document.get_document_operations() 198 | if not even_if_empty and len(operations) == 0: 199 | return 200 | 201 | payload = InteragentProtocolUtils.serialize(NewOperations( 202 | operations_list=json.dumps(operations) 203 | )) 204 | io_data = self._gateway.generate_io_data( 205 | payload, 206 | connection.get_active_address(), 207 | ) 208 | self._gateway.write_io_data( 209 | io_data, 210 | reliability=True, 211 | ) 212 | 213 | def stop(self): 214 | connections = ConnectionStore.get_instance().get_open_connections() 215 | io_data = self._gateway.generate_io_data( 216 | InteragentProtocolUtils.serialize(Bye()), 217 | [connection.get_active_address() for connection in connections], 218 | ) 219 | self._gateway.write_io_data(io_data) 220 | ConnectionStore.reset_instance() 221 | -------------------------------------------------------------------------------- /agent/tandem/agent/protocol/handlers/rendezvous.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import json 3 | import uuid 4 | from tandem.agent.configuration import USE_RELAY 5 | from tandem.agent.models.connection import HolePunchedConnection 6 | from tandem.agent.stores.connection import ConnectionStore 7 | from tandem.agent.utils.hole_punching import HolePunchingUtils 8 | from tandem.shared.models.peer import Peer 9 | from tandem.shared.protocol.handlers.addressed import AddressedHandler 10 | from tandem.shared.protocol.messages.rendezvous import ( 11 | RendezvousProtocolUtils, 12 | RendezvousProtocolMessageType, 13 | ) 14 | from tandem.agent.protocol.messages.interagent import ( 15 | InteragentProtocolUtils, 16 | NewOperations, 17 | ) 18 | from tandem.shared.utils.static_value import static_value as staticvalue 19 | from tandem.agent.models.connection_state import ConnectionState 20 | 21 | 22 | class RendezvousProtocolHandler(AddressedHandler): 23 | @staticvalue 24 | def _protocol_message_utils(self): 25 | return RendezvousProtocolUtils 26 | 27 | @staticvalue 28 | def _protocol_message_handlers(self): 29 | return { 30 | RendezvousProtocolMessageType.SetupParameters.value: 31 | self._handle_setup_parameters, 32 | RendezvousProtocolMessageType.Error.value: 33 | self._handle_error, 34 | } 35 | 36 | def __init__(self, id, gateway, time_scheduler, document): 37 | self._id = id 38 | self._gateway = gateway 39 | self._time_scheduler = time_scheduler 40 | self._document = document 41 | 42 | def _handle_setup_parameters(self, message, sender_address): 43 | public_address = (message.public[0], message.public[1]) 44 | private_address = (message.private[0], message.private[1]) 45 | logging.debug( 46 | "Received SetupParameters - Connect to {} at public {}:{} " 47 | "and private {}:{}" 48 | .format(message.peer_id, *public_address, *private_address), 49 | ) 50 | peer = Peer( 51 | id=uuid.UUID(message.peer_id), 52 | public_address=public_address, 53 | private_address=private_address, 54 | ) 55 | new_connection = HolePunchedConnection( 56 | peer=peer, 57 | initiated_connection=message.initiate, 58 | ) 59 | new_connection.set_interval_handle(self._time_scheduler.run_every( 60 | HolePunchingUtils.PING_INTERVAL, 61 | HolePunchingUtils.generate_send_ping( 62 | self._gateway, 63 | peer.get_addresses(), 64 | self._id, 65 | ), 66 | )) 67 | 68 | def handle_hole_punching_timeout(connection): 69 | if connection.get_connection_state() == ConnectionState.OPEN: 70 | return 71 | 72 | if not USE_RELAY: 73 | logging.info( 74 | "Connection {} is unreachable. Not switching to RELAY " 75 | "because it was disabled." 76 | .format(connection.get_peer().get_public_address()), 77 | ) 78 | connection.set_connection_state(ConnectionState.UNREACHABLE) 79 | return 80 | 81 | logging.info("Switching connection {} to RELAY".format( 82 | connection.get_peer().get_public_address() 83 | )) 84 | 85 | connection.set_connection_state(ConnectionState.RELAY) 86 | 87 | operations = self._document.get_document_operations() 88 | payload = InteragentProtocolUtils.serialize(NewOperations( 89 | operations_list=json.dumps(operations) 90 | )) 91 | io_data = self._gateway.generate_io_data( 92 | payload, 93 | connection.get_peer().get_public_address(), 94 | ) 95 | self._gateway.write_io_data( 96 | io_data, 97 | reliability=True, 98 | ) 99 | 100 | self._time_scheduler.run_after( 101 | HolePunchingUtils.TIMEOUT, 102 | handle_hole_punching_timeout, 103 | new_connection 104 | ) 105 | ConnectionStore.get_instance().add_connection(new_connection) 106 | 107 | def _handle_error(self, message, sender_address): 108 | logging.info("Rendezvous Error: {}".format(message.message)) 109 | -------------------------------------------------------------------------------- /agent/tandem/agent/protocol/messages/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/agent/tandem/agent/protocol/messages/__init__.py -------------------------------------------------------------------------------- /agent/tandem/agent/protocol/messages/editor.py: -------------------------------------------------------------------------------- 1 | import json 2 | import enum 3 | 4 | 5 | class EditorProtocolMarshalError(ValueError): 6 | pass 7 | 8 | 9 | class EditorProtocolMessageType(enum.Enum): 10 | ApplyText = "apply-text" 11 | ApplyPatches = "apply-patches" 12 | CheckDocumentSync = "check-document-sync" 13 | ConnectTo = "connect-to" 14 | HostSession = "host-session" 15 | JoinSession = "join-session" 16 | NewPatches = "new-patches" 17 | SessionInfo = "session-info" 18 | UserChangedEditorText = "user-changed-editor-text" 19 | WriteRequest = "write-request" 20 | WriteRequestAck = "write-request-ack" 21 | 22 | 23 | class UserChangedEditorText: 24 | """ 25 | Sent by the editor plugin to the agent to 26 | notify it that the user changed the text buffer. 27 | """ 28 | def __init__(self, contents): 29 | self.type = EditorProtocolMessageType.UserChangedEditorText 30 | self.contents = contents 31 | 32 | def to_payload(self): 33 | return { 34 | "contents": self.contents, 35 | } 36 | 37 | @staticmethod 38 | def from_payload(payload): 39 | return UserChangedEditorText(payload["contents"]) 40 | 41 | 42 | class CheckDocumentSync: 43 | """ 44 | Sent by the editor plugin to the agent to 45 | check whether the editor and the crdt have their 46 | document contents in sync 47 | """ 48 | def __init__(self, contents): 49 | self.type = EditorProtocolMessageType.CheckDocumentSync 50 | self.contents = contents 51 | 52 | def to_payload(self): 53 | return { 54 | "contents": self.contents, 55 | } 56 | 57 | @staticmethod 58 | def from_payload(payload): 59 | return CheckDocumentSync(payload["contents"]) 60 | 61 | 62 | class ApplyText: 63 | """ 64 | Sent by the agent to the editor plugin to 65 | notify it that someone else edited the text buffer. 66 | """ 67 | def __init__(self, contents): 68 | self.type = EditorProtocolMessageType.ApplyText 69 | self.contents = contents 70 | 71 | def to_payload(self): 72 | return { 73 | "contents": self.contents, 74 | } 75 | 76 | @staticmethod 77 | def from_payload(payload): 78 | return ApplyText(payload["contents"]) 79 | 80 | 81 | class ConnectTo: 82 | """ 83 | Sent by the plugin to the agent to tell it to connect 84 | to another agent. 85 | """ 86 | def __init__(self, host, port): 87 | self.type = EditorProtocolMessageType.ConnectTo 88 | self.host = host 89 | self.port = port 90 | 91 | def to_payload(self): 92 | return { 93 | "host": self.host, 94 | "port": self.port, 95 | } 96 | 97 | @staticmethod 98 | def from_payload(payload): 99 | return ConnectTo(payload["host"], payload["port"]) 100 | 101 | 102 | class WriteRequest: 103 | """ 104 | Sent by the agent to the plugin to request for the ability 105 | to apply remote operations to the CRDT. 106 | """ 107 | def __init__(self, seq): 108 | self.type = EditorProtocolMessageType.WriteRequest 109 | self.seq = seq 110 | 111 | def to_payload(self): 112 | return { 113 | "seq": self.seq, 114 | } 115 | 116 | @staticmethod 117 | def from_payload(payload): 118 | return WriteRequest(payload["seq"]) 119 | 120 | 121 | class WriteRequestAck: 122 | """ 123 | Sent by the plugin to the agent in response to a WriteRequest 124 | message to grant it permission to apply remote operations to the CRDT. 125 | 126 | By sending this message the plugin agrees to not allow users 127 | to modify their local buffer until the remote operations have been 128 | sent back to the plugin via an ApplyPatches message. 129 | """ 130 | def __init__(self, seq): 131 | self.type = EditorProtocolMessageType.WriteRequestAck 132 | self.seq = seq 133 | 134 | def to_payload(self): 135 | return { 136 | "seq": self.seq, 137 | } 138 | 139 | @staticmethod 140 | def from_payload(payload): 141 | return WriteRequestAck(payload["seq"]) 142 | 143 | 144 | class NewPatches: 145 | """ 146 | Sent by the plugin to the agent to inform it of changes made by 147 | the user to their local text buffer. 148 | 149 | patch_list should be a list of dictionaries where each dictionary 150 | represents a change that the user made to their local text buffer. 151 | The patches should be ordered such that they are applied in the 152 | correct order when the list is traversed from front to back. 153 | 154 | Each patch should have the form: 155 | 156 | { 157 | "start": {"row": , "column": }, 158 | "end": {"row": , "column": }, 159 | "text": , 160 | } 161 | """ 162 | def __init__(self, patch_list): 163 | self.type = EditorProtocolMessageType.NewPatches 164 | self.patch_list = patch_list 165 | 166 | def to_payload(self): 167 | return { 168 | "patch_list": self.patch_list 169 | } 170 | 171 | @staticmethod 172 | def from_payload(payload): 173 | return NewPatches(payload["patch_list"]) 174 | 175 | 176 | class ApplyPatches: 177 | """ 178 | Sent by the agent to the plugin to inform it of remote changes 179 | that should be applied to their local text buffer. 180 | 181 | patch_list will be a list of dictionaries where each dictionary 182 | represents a change that some remote user made to the text buffer. 183 | The order of the patches is significant. They should applied in 184 | the order they are found in this message. 185 | 186 | Each patch will have the form: 187 | 188 | { 189 | "oldStart": {"row": , "column": }, 190 | "oldEnd": {"row": , "column": }, 191 | "oldText": , 192 | "newStart": {"row": , "column": }, 193 | "newEnd": {"row": , "column": }, 194 | "newText": , 195 | } 196 | """ 197 | def __init__(self, patch_list): 198 | self.type = EditorProtocolMessageType.ApplyPatches 199 | self.patch_list = patch_list 200 | 201 | def to_payload(self): 202 | return { 203 | "patch_list": self.patch_list 204 | } 205 | 206 | @staticmethod 207 | def from_payload(payload): 208 | return ApplyPatches(payload["patch_list"]) 209 | 210 | 211 | class HostSession: 212 | """ 213 | Sent by the plugin to the agent to ask it to start hosting a new 214 | session. 215 | """ 216 | def __init__(self): 217 | self.type = EditorProtocolMessageType.HostSession 218 | 219 | def to_payload(self): 220 | return {} 221 | 222 | @staticmethod 223 | def from_payload(payload): 224 | return HostSession() 225 | 226 | 227 | class JoinSession: 228 | """ 229 | Sent by the plugin to the agent to ask it to join an existing 230 | session. 231 | """ 232 | def __init__(self, session_id): 233 | self.type = EditorProtocolMessageType.JoinSession 234 | self.session_id = session_id 235 | 236 | def to_payload(self): 237 | return { 238 | "session_id": str(self.session_id), 239 | } 240 | 241 | @staticmethod 242 | def from_payload(payload): 243 | return JoinSession(payload["session_id"]) 244 | 245 | 246 | class SessionInfo: 247 | """ 248 | Sent by the agent to the plugin to pass it the session ID. 249 | """ 250 | def __init__(self, session_id): 251 | self.type = EditorProtocolMessageType.SessionInfo 252 | self.session_id = session_id 253 | 254 | def to_payload(self): 255 | return { 256 | "session_id": str(self.session_id), 257 | } 258 | 259 | @staticmethod 260 | def from_payload(payload): 261 | return SessionInfo(payload["session_id"]) 262 | 263 | 264 | def serialize(message): 265 | as_dict = { 266 | "type": message.type.value, 267 | "payload": message.to_payload(), 268 | "version": 1, 269 | } 270 | return json.dumps(as_dict) 271 | 272 | 273 | def deserialize(data): 274 | try: 275 | as_dict = json.loads(data) 276 | message_type = as_dict["type"] 277 | payload = as_dict["payload"] 278 | 279 | if message_type == EditorProtocolMessageType.ConnectTo.value: 280 | return ConnectTo.from_payload(payload) 281 | 282 | elif message_type == EditorProtocolMessageType.WriteRequest.value: 283 | return WriteRequest.from_payload(payload) 284 | 285 | elif message_type == EditorProtocolMessageType.WriteRequestAck.value: 286 | return WriteRequestAck.from_payload(payload) 287 | 288 | elif message_type == \ 289 | EditorProtocolMessageType.UserChangedEditorText.value: 290 | return UserChangedEditorText.from_payload(payload) 291 | 292 | elif message_type == EditorProtocolMessageType.ApplyText.value: 293 | return ApplyText.from_payload(payload) 294 | 295 | elif message_type == EditorProtocolMessageType.NewPatches.value: 296 | return NewPatches.from_payload(payload) 297 | 298 | elif message_type == EditorProtocolMessageType.ApplyPatches.value: 299 | return ApplyPatches.from_payload(payload) 300 | 301 | elif message_type == EditorProtocolMessageType.CheckDocumentSync.value: 302 | return CheckDocumentSync.from_payload(payload) 303 | 304 | elif message_type == EditorProtocolMessageType.HostSession.value: 305 | return HostSession.from_payload(payload) 306 | 307 | elif message_type == EditorProtocolMessageType.JoinSession.value: 308 | return JoinSession.from_payload(payload) 309 | 310 | elif message_type == EditorProtocolMessageType.SessionInfo.value: 311 | return SessionInfo.from_payload(payload) 312 | 313 | else: 314 | raise EditorProtocolMarshalError 315 | 316 | except: 317 | raise EditorProtocolMarshalError 318 | -------------------------------------------------------------------------------- /agent/tandem/agent/protocol/messages/interagent.py: -------------------------------------------------------------------------------- 1 | from tandem.shared.protocol.messages.base import ( 2 | ProtocolMessageTypeBase, 3 | ProtocolMessageBase, 4 | ProtocolUtilsBase, 5 | ) 6 | from tandem.shared.utils.static_value import static_value as staticvalue 7 | 8 | 9 | class InteragentProtocolMessageType(ProtocolMessageTypeBase): 10 | # Connection setup messages 11 | Ping = "ia-ping" 12 | PingBack = "ia-ping-back" 13 | Syn = "ia-syn" 14 | Hello = "ia-hello" 15 | 16 | # Regular interagent messages 17 | NewOperations = "ia-new-operations" 18 | Bye = "ia-bye" 19 | 20 | 21 | class Ping(ProtocolMessageBase): 22 | """ 23 | Sent by the agent to a peer to maintain or open a connection. 24 | """ 25 | def __init__(self, **kwargs): 26 | super(Ping, self).__init__( 27 | InteragentProtocolMessageType.Ping, 28 | **kwargs, 29 | ) 30 | 31 | @staticvalue 32 | def _payload_keys(self): 33 | return ["id"] 34 | 35 | 36 | class PingBack(ProtocolMessageBase): 37 | """ 38 | Sent in response to a Ping message to acknowledge receipt. 39 | """ 40 | def __init__(self, **kwargs): 41 | super(PingBack, self).__init__( 42 | InteragentProtocolMessageType.PingBack, 43 | **kwargs, 44 | ) 45 | 46 | @staticvalue 47 | def _payload_keys(self): 48 | return ["id"] 49 | 50 | 51 | class Syn(ProtocolMessageBase): 52 | """ 53 | Sent by the connection initiator to indicate that it has 54 | completed its connection set up and wishes to begin 55 | communicating via regular protocol messages. 56 | 57 | The initiator should continue sending this message until 58 | it receives a regular protocol message from the non-initiator. 59 | """ 60 | def __init__(self, **kwargs): 61 | super(Syn, self).__init__( 62 | InteragentProtocolMessageType.Syn, 63 | **kwargs, 64 | ) 65 | 66 | @staticvalue 67 | def _payload_keys(self): 68 | return [] 69 | 70 | 71 | class Hello(ProtocolMessageBase): 72 | """ 73 | Sent directly from one agent to another to introduce itself. 74 | 75 | This message is used to directly establish a connection. It 76 | is sent after receiving a ConnectTo message from the plugin. 77 | 78 | The should_reply flag is set if the agent wants the remote 79 | peer to respond with a Hello message containing its ID. 80 | """ 81 | def __init__(self, **kwargs): 82 | super(Hello, self).__init__( 83 | InteragentProtocolMessageType.Hello, 84 | **kwargs, 85 | ) 86 | 87 | @staticvalue 88 | def _payload_keys(self): 89 | return ["id", "should_reply"] 90 | 91 | 92 | class Bye(ProtocolMessageBase): 93 | def __init__(self, **kwargs): 94 | super(Bye, self).__init__( 95 | InteragentProtocolMessageType.Bye, 96 | **kwargs, 97 | ) 98 | 99 | @staticvalue 100 | def _payload_keys(self): 101 | return [] 102 | 103 | 104 | class NewOperations(ProtocolMessageBase): 105 | """ 106 | Sent to other agents to notify them of new CRDT operations to apply. 107 | """ 108 | def __init__(self, **kwargs): 109 | super(NewOperations, self).__init__( 110 | InteragentProtocolMessageType.NewOperations, 111 | **kwargs, 112 | ) 113 | 114 | @staticvalue 115 | def _payload_keys(self): 116 | return ['operations_list'] 117 | 118 | 119 | class InteragentProtocolUtils(ProtocolUtilsBase): 120 | @classmethod 121 | @staticvalue 122 | def _protocol_message_constructors(cls): 123 | return { 124 | InteragentProtocolMessageType.Ping.value: Ping, 125 | InteragentProtocolMessageType.PingBack.value: PingBack, 126 | InteragentProtocolMessageType.Syn.value: Syn, 127 | InteragentProtocolMessageType.Hello.value: Hello, 128 | InteragentProtocolMessageType.Bye.value: Bye, 129 | InteragentProtocolMessageType.NewOperations.value: NewOperations, 130 | } 131 | -------------------------------------------------------------------------------- /agent/tandem/agent/stores/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/agent/tandem/agent/stores/__init__.py -------------------------------------------------------------------------------- /agent/tandem/agent/stores/connection.py: -------------------------------------------------------------------------------- 1 | from tandem.shared.stores.base import StoreBase 2 | from tandem.agent.models.connection_state import ConnectionState 3 | 4 | 5 | class ConnectionStore(StoreBase): 6 | def __init__(self): 7 | self._connections = {} 8 | 9 | def add_connection(self, connection): 10 | self._connections[connection.get_id()] = connection 11 | 12 | def remove_connection(self, connection): 13 | del self._connections[connection.get_id()] 14 | 15 | def get_connection_by_id(self, id): 16 | return self._connections.get(id, None) 17 | 18 | def get_connection_by_address(self, address): 19 | for _, connection in self._connections.items(): 20 | if connection.get_active_address() == address: 21 | return connection 22 | return None 23 | 24 | def get_open_connections(self): 25 | return [ 26 | connection for _, connection in self._connections.items() 27 | if ( 28 | connection.get_connection_state() == ConnectionState.OPEN or 29 | connection.get_connection_state() == ConnectionState.RELAY 30 | ) 31 | ] 32 | -------------------------------------------------------------------------------- /agent/tandem/agent/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/agent/tandem/agent/utils/__init__.py -------------------------------------------------------------------------------- /agent/tandem/agent/utils/hole_punching.py: -------------------------------------------------------------------------------- 1 | from tandem.agent.protocol.messages.interagent import ( 2 | InteragentProtocolUtils, 3 | Ping, 4 | Syn, 5 | ) 6 | 7 | 8 | class HolePunchingUtils: 9 | PING_INTERVAL = 0.15 10 | SYN_INTERVAL = 0.15 11 | TIMEOUT = 3 12 | 13 | @staticmethod 14 | def generate_send_ping(gateway, addresses, id): 15 | def send_ping(): 16 | HolePunchingUtils._send_message( 17 | gateway, 18 | addresses, 19 | Ping(id=str(id)), 20 | ) 21 | return send_ping 22 | 23 | @staticmethod 24 | def generate_send_syn(gateway, address): 25 | def send_syn(): 26 | HolePunchingUtils._send_message( 27 | gateway, 28 | address, 29 | Syn(), 30 | ) 31 | return send_syn 32 | 33 | @staticmethod 34 | def _send_message(gateway, addresses, message): 35 | io_data = gateway.generate_io_data( 36 | InteragentProtocolUtils.serialize(message), 37 | addresses, 38 | ) 39 | gateway.write_io_data(io_data) 40 | -------------------------------------------------------------------------------- /agent/tandem/shared: -------------------------------------------------------------------------------- 1 | ../../lib-python/ -------------------------------------------------------------------------------- /agent/test_client.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import random 4 | from subprocess import Popen, PIPE 5 | import tandem.agent.protocol.messages.editor as m 6 | from tandem.agent.configuration import BASE_DIR 7 | 8 | 9 | def start_agent(extra_args=None): 10 | if extra_args is None: 11 | extra_args = [] 12 | return Popen( 13 | ["python3", os.path.join(BASE_DIR, "main.py")] + extra_args, 14 | stdin=PIPE, 15 | stdout=PIPE, 16 | encoding="utf-8", 17 | ) 18 | 19 | 20 | def send_user_changed(agent_stdin, text): 21 | message = m.UserChangedEditorText(text) 22 | agent_stdin.write(m.serialize(message)) 23 | agent_stdin.write("\n") 24 | agent_stdin.flush() 25 | 26 | 27 | def send_new_patches(agent_stdin, start, end, text): 28 | patches = m.NewPatches([{ 29 | "start": start, 30 | "end": end, 31 | "text": text, 32 | }]) 33 | agent_stdin.write(m.serialize(patches)) 34 | agent_stdin.write("\n") 35 | agent_stdin.flush() 36 | 37 | 38 | def send_request_write_ack(agent_stdin, seq): 39 | agent_stdin.write(m.serialize(m.WriteRequestAck(seq))) 40 | agent_stdin.write("\n") 41 | agent_stdin.flush() 42 | 43 | 44 | def print_raw_message(agent_stdout): 45 | resp = agent_stdout.readline() 46 | print("Received: " + resp) 47 | 48 | 49 | def extract_message(agent_stdout): 50 | resp = agent_stdout.readline() 51 | return m.deserialize(resp) 52 | 53 | 54 | def get_string_ports(): 55 | starting_port = random.randint(60600, 62600) 56 | port1 = str(starting_port) 57 | port2 = str(starting_port + 1) 58 | return port1, port2 59 | 60 | 61 | def ping_test(): 62 | """ 63 | Starts 2 agents and checks that they can establish 64 | a connection to eachother and exchange a ping message. 65 | """ 66 | agent1_port, agent2_port = get_string_ports() 67 | 68 | agent1 = start_agent(["--port", agent1_port]) 69 | agent2 = start_agent([ 70 | "--port", 71 | agent2_port, 72 | "--log-file", 73 | "/tmp/tandem-agent-2.log", 74 | ]) 75 | 76 | # Wait for the agents to start accepting connections 77 | time.sleep(1) 78 | 79 | message = m.ConnectTo("localhost", int(agent1_port)) 80 | agent2.stdin.write(m.serialize(message)) 81 | agent2.stdin.write("\n") 82 | agent2.stdin.flush() 83 | 84 | # Wait for the pings 85 | time.sleep(2) 86 | 87 | agent1.stdin.close() 88 | agent1.terminate() 89 | agent2.stdin.close() 90 | agent2.terminate() 91 | 92 | agent1.wait() 93 | agent2.wait() 94 | 95 | 96 | def text_transfer_test(): 97 | """ 98 | Tests the Milestone 1 flow by starting 2 agents and 99 | transfering text data from one agent to the other. 100 | 101 | 1. Instruct agent 2 to connect to agent 1 102 | 2. Send a "text changed" message to agent 1 103 | (simulating what the plugin would do) 104 | 3. Expect an "apply text" message to be "output" by agent 2 105 | (this would be an instruction to the plugin to change 106 | the editor's text buffer) 107 | """ 108 | agent1_port, agent2_port = get_string_ports() 109 | 110 | agent1 = start_agent(["--port", agent1_port]) 111 | agent2 = start_agent([ 112 | "--port", 113 | agent2_port, 114 | "--log-file", 115 | "/tmp/tandem-agent-2.log", 116 | ]) 117 | 118 | # Wait for the agents to start accepting connections 119 | time.sleep(1) 120 | 121 | message = m.ConnectTo("localhost", int(agent1_port)) 122 | agent2.stdin.write(m.serialize(message)) 123 | agent2.stdin.write("\n") 124 | agent2.stdin.flush() 125 | 126 | # Wait for the pings 127 | time.sleep(2) 128 | 129 | # Simulate a text buffer change - the plugin notifes agent1 that 130 | # the text buffer has changed 131 | send_user_changed(agent1.stdin, ["Hello world!"]) 132 | 133 | # Expect agent2 to receive a ApplyText message 134 | print_raw_message(agent2.stdout) 135 | 136 | # Repeat 137 | send_user_changed(agent1.stdin, ["Hello world again!"]) 138 | print_raw_message(agent2.stdout) 139 | 140 | # Shut down the agents 141 | agent1.stdin.close() 142 | agent1.terminate() 143 | agent2.stdin.close() 144 | agent2.terminate() 145 | 146 | agent1.wait() 147 | agent2.wait() 148 | 149 | 150 | def crdt_test(): 151 | """ 152 | Tests the Milestone 2 flow. 153 | 1. Agent 1 makes a local change 154 | 2. Check that Agent 2 received the changes 155 | 3. Repeat 156 | 4. Agent 2 makes a local change 157 | 5. Check that Agent 1 received the changes 158 | """ 159 | agent1_port, agent2_port = get_string_ports() 160 | 161 | agent1 = start_agent(["--port", agent1_port]) 162 | agent2 = start_agent([ 163 | "--port", 164 | agent2_port, 165 | "--log-file", 166 | "/tmp/tandem-agent-2.log", 167 | ]) 168 | 169 | # Wait for the agents to start accepting connections 170 | time.sleep(1) 171 | 172 | message = m.ConnectTo("localhost", int(agent1_port)) 173 | agent2.stdin.write(m.serialize(message)) 174 | agent2.stdin.write("\n") 175 | agent2.stdin.flush() 176 | 177 | # Wait for connection 178 | time.sleep(1) 179 | 180 | # Simulate a text buffer change - the plugin notifes agent1 that 181 | # the text buffer has changed 182 | send_new_patches( 183 | agent1.stdin, 184 | {"row": 0, "column": 0}, 185 | {"row": 0, "column": 0}, 186 | "Hello world!", 187 | ) 188 | print("Agent 1 made an edit") 189 | 190 | # Simulate a text buffer change - the plugin notifes agent1 that 191 | # the text buffer has changed 192 | send_new_patches( 193 | agent1.stdin, 194 | {"row": 0, "column": 12}, 195 | {"row": 0, "column": 0}, 196 | " Hello world again!", 197 | ) 198 | print("Agent 1 made a second edit") 199 | 200 | # The agent should not resend the request write message 201 | time.sleep(1) 202 | 203 | # Expect agent2 to receive a "Request Write" message 204 | print_raw_message(agent2.stdout) 205 | 206 | # Allow the plugin to apply the remote changes 207 | send_request_write_ack(agent2.stdin, 0) 208 | 209 | # Expect agent2 to get the changes 210 | print_raw_message(agent2.stdout) 211 | 212 | # Simulate an edit that occurs on agent2's machine 213 | send_new_patches( 214 | agent2.stdin, 215 | {"row": 0, "column": 0}, 216 | {"row": 0, "column": 0}, 217 | "Agent 2 says hi! ", 218 | ) 219 | print("Agent 2 made an edit") 220 | 221 | # Expect agent1 to receive a RequestWrite message 222 | print_raw_message(agent1.stdout) 223 | 224 | # Allow changes to be applied 225 | send_request_write_ack(agent1.stdin, 0) 226 | 227 | # Expect to receive the text patches 228 | print_raw_message(agent1.stdout) 229 | 230 | # Simulate an edit that occurs on agent2's machine 231 | send_new_patches( 232 | agent2.stdin, 233 | {"row": 0, "column": 0}, 234 | {"row": 0, "column": 0}, 235 | "Agent 2 says hi again! ", 236 | ) 237 | print("Agent 2 made a second edit!") 238 | 239 | # Expect agent1 to receive a RequestWrite message 240 | print_raw_message(agent1.stdout) 241 | 242 | # Allow changes to be applied 243 | send_request_write_ack(agent1.stdin, 1) 244 | 245 | # Expect to receive the text patches 246 | print_raw_message(agent1.stdout) 247 | 248 | time.sleep(2) 249 | 250 | # Shut down the agents 251 | agent1.stdin.close() 252 | agent1.terminate() 253 | agent2.stdin.close() 254 | agent2.terminate() 255 | 256 | agent1.wait() 257 | agent2.wait() 258 | 259 | 260 | def hole_punch_test(): 261 | agent1_port, agent2_port = get_string_ports() 262 | agent3_port = str(int(agent2_port) + 1) 263 | 264 | agent1 = start_agent(["--port", agent1_port]) 265 | agent2 = start_agent([ 266 | "--port", 267 | agent2_port, 268 | "--log-file", 269 | "/tmp/tandem-agent-2.log", 270 | ]) 271 | agent3 = start_agent([ 272 | "--port", 273 | agent3_port, 274 | "--log-file", 275 | "/tmp/tandem-agent-3.log", 276 | ]) 277 | 278 | # Wait for the agents to start up 279 | time.sleep(1) 280 | 281 | host_session = m.HostSession() 282 | agent1.stdin.write(m.serialize(host_session)) 283 | agent1.stdin.write("\n") 284 | agent1.stdin.flush() 285 | 286 | session_info = extract_message(agent1.stdout) 287 | print("Session ID: {}".format(session_info.session_id)) 288 | 289 | join_session = m.JoinSession(session_id=session_info.session_id) 290 | agent2.stdin.write(m.serialize(join_session)) 291 | agent2.stdin.write("\n") 292 | agent2.stdin.flush() 293 | agent3.stdin.write(m.serialize(join_session)) 294 | agent3.stdin.write("\n") 295 | agent3.stdin.flush() 296 | 297 | time.sleep(5) 298 | 299 | # Shut down the agents 300 | agent1.stdin.close() 301 | agent1.terminate() 302 | agent2.stdin.close() 303 | agent2.terminate() 304 | agent3.stdin.close() 305 | agent3.terminate() 306 | 307 | agent1.wait() 308 | agent2.wait() 309 | agent3.wait() 310 | 311 | 312 | def main(): 313 | hole_punch_test() 314 | 315 | 316 | if __name__ == "__main__": 317 | main() 318 | -------------------------------------------------------------------------------- /crdt/.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | build 3 | webpack.config.js 4 | -------------------------------------------------------------------------------- /crdt/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "airbnb", 3 | "rules": { 4 | "indent": [error, 4] 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /crdt/api/apply_operations.js: -------------------------------------------------------------------------------- 1 | import DocumentStore from '../stores/document'; 2 | 3 | /* 4 | * API Name: applyOperations 5 | * Returns: Plugin changes that need to be applied to show non-local changes 6 | * Used for applying CRDT operations to the local document 7 | */ 8 | export default (operations) => { 9 | const { textUpdates } = DocumentStore.getDocument().integrateOperations(operations); 10 | return textUpdates; 11 | }; 12 | -------------------------------------------------------------------------------- /crdt/api/get_document_operations.js: -------------------------------------------------------------------------------- 1 | import DocumentStore from '../stores/document'; 2 | 3 | /* 4 | * API Name: getDocumentOperations 5 | * Returns: An array of operations to get to where the document is currently 6 | * Used for setting up a new agent 7 | */ 8 | export default () => DocumentStore.getDocument().getOperations(); 9 | -------------------------------------------------------------------------------- /crdt/api/get_document_text.js: -------------------------------------------------------------------------------- 1 | import DocumentStore from '../stores/document'; 2 | 3 | /* 4 | * API Name: getDocumentText 5 | * Returns: The text value of the document's contents 6 | * Used for retrieving the text contents of the document 7 | */ 8 | export default () => DocumentStore.getDocument().getText(); 9 | -------------------------------------------------------------------------------- /crdt/api/index.js: -------------------------------------------------------------------------------- 1 | import applyOperations from './apply_operations'; 2 | import getDocumentText from './get_document_text'; 3 | import getDocumentOperations from './get_document_operations'; 4 | import replicateDocument from './replicate_document'; 5 | import setDocument from './set_document'; 6 | import setTextInRange from './set_text_in_range'; 7 | 8 | export default { 9 | applyOperations, 10 | getDocumentText, 11 | getDocumentOperations, 12 | replicateDocument, 13 | setDocument, 14 | setTextInRange, 15 | }; 16 | -------------------------------------------------------------------------------- /crdt/api/replicate_document.js: -------------------------------------------------------------------------------- 1 | import uuid from 'uuid'; 2 | import DocumentStore from '../stores/document'; 3 | 4 | /* 5 | * API Name: replicateDocument 6 | * Returns: A replica of the local document 7 | * Used for sending the local document to a new agent 8 | */ 9 | export default () => DocumentStore.getDocument().replicate(uuid()); 10 | -------------------------------------------------------------------------------- /crdt/api/set_document.js: -------------------------------------------------------------------------------- 1 | import DocumentStore from '../stores/document'; 2 | 3 | /* 4 | * API Name: setDocument 5 | * Returns: null 6 | * Used for assigning the local document to a new document 7 | */ 8 | export default (newDocument) => { 9 | DocumentStore.setDocument(newDocument); 10 | }; 11 | -------------------------------------------------------------------------------- /crdt/api/set_text_in_range.js: -------------------------------------------------------------------------------- 1 | import DocumentStore from '../stores/document'; 2 | 3 | /* 4 | * API Name: setTextInRange 5 | * Returns: An array of CRDT operations to be broadcasted to the other documents 6 | * Used for applying the plugin changes to the CRDT document 7 | */ 8 | export default (start, end, text, options) => 9 | DocumentStore.getDocument().setTextInRange(start, end, text, options); 10 | -------------------------------------------------------------------------------- /crdt/index.js: -------------------------------------------------------------------------------- 1 | import uuid from 'uuid'; 2 | import { Document } from '@atom/teletype-crdt'; 3 | import DocumentStore from './stores/document'; 4 | import api from './api'; 5 | import io from './io'; 6 | import logger from './utils/logger'; 7 | 8 | const localDocument = new Document({ siteId: uuid(), text: '' }); 9 | DocumentStore.setDocument(localDocument); 10 | 11 | const processPayload = (payload) => { 12 | try { 13 | logger.debug(`Processing payload: ${payload}`); 14 | const { function: fcn, parameters: params = [] } = JSON.parse(payload); 15 | 16 | logger.debug(`Calling api function: ${JSON.stringify(fcn)}, with parameters: ${JSON.stringify(params)}`); 17 | const res = { value: api[fcn](...params) }; 18 | const result = JSON.stringify(res); 19 | 20 | logger.debug(`Finished processing, with result: ${result}`); 21 | io.write(result); 22 | } catch (error) { 23 | logger.error(`There has been an error in processing payload: ${payload}`); 24 | logger.error(`Error: ${JSON.stringify(error)}`); 25 | } 26 | }; 27 | 28 | io.addIoHandler(processPayload); 29 | 30 | logger.info('Tandem-CRDT is up and running'); 31 | -------------------------------------------------------------------------------- /crdt/io/index.js: -------------------------------------------------------------------------------- 1 | const ioHandlers = []; 2 | let currentPayload = ''; 3 | 4 | process.stdin.resume(); 5 | process.stdin.setEncoding('utf-8'); 6 | process.stdin.on('data', (data) => { 7 | const payloadList = (currentPayload + data).split('\n'); 8 | currentPayload = payloadList.pop(); 9 | payloadList.forEach((payload) => { 10 | ioHandlers.forEach(handler => setImmediate(() => handler(payload))); 11 | }); 12 | }); 13 | 14 | const addIoHandler = handler => ioHandlers.push(handler); 15 | const write = (outPayload) => { 16 | process.stdout.write(outPayload, 'utf-8'); 17 | process.stdout.write('\n'); 18 | }; 19 | 20 | export default { 21 | addIoHandler, 22 | write, 23 | }; 24 | -------------------------------------------------------------------------------- /crdt/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tandem-crdt", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "watch": "$(npm bin)/webpack --progress --colors --watch", 8 | "build": "$(npm bin)/webpack --progress --colors", 9 | "start": "node ./build/bundle.js", 10 | "clean": "rm -r ./build", 11 | "lint": "$(npm bin)/eslint .", 12 | "test": "echo \"Error: no test specified\" && exit 1" 13 | }, 14 | "author": "", 15 | "license": "ISC", 16 | "dependencies": { 17 | "@atom/teletype-crdt": "0.8.1", 18 | "uuid": "^3.1.0", 19 | "winston": "^3.0.0-rc1" 20 | }, 21 | "devDependencies": { 22 | "babel-core": "^6.26.0", 23 | "babel-loader": "^7.1.2", 24 | "eslint": "^4.15.0", 25 | "eslint-config-airbnb": "^16.1.0", 26 | "eslint-plugin-import": "^2.8.0", 27 | "eslint-plugin-jsx-a11y": "^6.0.3", 28 | "eslint-plugin-react": "^7.5.1", 29 | "webpack": "^3.10.0" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /crdt/stores/document.js: -------------------------------------------------------------------------------- 1 | let localDocument = null; 2 | 3 | export default { 4 | getDocument: () => localDocument, 5 | setDocument: (newDocument) => { 6 | localDocument = newDocument; 7 | }, 8 | }; 9 | -------------------------------------------------------------------------------- /crdt/utils/logger.js: -------------------------------------------------------------------------------- 1 | import winston from 'winston'; 2 | 3 | const prefix = Date.now(); 4 | const format = winston.format.printf(info => `[${info.timestamp}] [${info.level}]: ${info.message}`); 5 | 6 | export default winston.createLogger({ 7 | format: winston.format.combine(winston.format.label(), winston.format.timestamp(), format), 8 | transports: [ 9 | new winston.transports.File({ filename: `/tmp/tandem-crdt-${prefix}.log` }), 10 | new winston.transports.File({ filename: `/tmp/tandem-crdt-debug-${prefix}.log`, level: 'debug' }), 11 | ], 12 | }); 13 | -------------------------------------------------------------------------------- /crdt/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | module.exports = { 4 | entry: [ 5 | './index.js' 6 | ], 7 | 8 | target: 'node', 9 | 10 | output: { 11 | path: path.resolve(__dirname, 'build'), 12 | filename: 'bundle.js' 13 | }, 14 | 15 | module: { 16 | loaders: [ 17 | { test: /\.js$/, exclude: /node_modules/, loader: 'babel-loader'}, 18 | ] 19 | } 20 | }; 21 | -------------------------------------------------------------------------------- /docs/CNAME: -------------------------------------------------------------------------------- 1 | typeintandem.com 2 | -------------------------------------------------------------------------------- /docs/img/neovim.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/docs/img/neovim.png -------------------------------------------------------------------------------- /docs/img/preview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/docs/img/preview.png -------------------------------------------------------------------------------- /docs/img/sublime.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/docs/img/sublime.png -------------------------------------------------------------------------------- /docs/img/vim.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/docs/img/vim.png -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 39 | 40 | Tandem - Decentralized, cross-editor, collaborative text-editing 41 | 42 | 43 | 44 |
45 |
TANDEM
46 |
Decentralized, cross-editor, collaborative text-editing 47 |
48 |
49 |
50 |
Tandem is a decentralized, collaborative text-editing solution.
51 |
Collaborating is as easy as installing the plugin on your editor and creating a Tandem Session.
52 |
Invite other people to your session, and get typing in tandem!
53 |
54 |
55 | 56 |
57 |
58 |
Full Guide available on GitHub:
59 | 60 |
61 |
62 |
Download Plugins
63 |
64 |
65 |
Neovim
66 | 67 |
Install
68 |
69 | README on GitHub 70 |
71 |
72 |
73 |
Sublime Text 3
74 | 75 |
Install
76 |
77 | README on GitHub 78 |
79 |
80 |
81 |
Vim
82 | 83 |
Install
84 |
85 | README on GitHub 86 |
87 |
88 |
89 |
90 |
91 |
Feedback
92 | 93 |
94 |
95 |
Created for the University of Waterloo's
96 |
Software Engineering Capstone Design Project
97 |
98 |
© Team Lightly, 2018
99 | 100 | 101 | -------------------------------------------------------------------------------- /lib-python/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/lib-python/__init__.py -------------------------------------------------------------------------------- /lib-python/io/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/lib-python/io/__init__.py -------------------------------------------------------------------------------- /lib-python/io/base.py: -------------------------------------------------------------------------------- 1 | from threading import Thread 2 | from tandem.shared.utils.proxy import ProxyUtils 3 | 4 | 5 | class InterfaceDataBase(object): 6 | def __init__(self, data): 7 | self._data = data 8 | 9 | def get_data(self): 10 | return self._data 11 | 12 | def is_empty(self): 13 | return self._data is None 14 | 15 | 16 | class InterfaceBase(object): 17 | data_class = InterfaceDataBase 18 | 19 | def __init__(self, incoming_data_handler, proxies=[]): 20 | self._incoming_data_handler = incoming_data_handler 21 | self._reader = Thread(target=self._read_data) 22 | self._proxies = proxies 23 | for proxy in proxies: 24 | proxy.attach_interface(self) 25 | 26 | def start(self): 27 | self._reader.start() 28 | 29 | def stop(self): 30 | self._reader.join() 31 | 32 | def generate_io_data(self, *args, **kwargs): 33 | new_args, new_kwargs = ProxyUtils.run( 34 | self._proxies, 35 | 'pre_generate_io_data', 36 | (args, kwargs), 37 | ) 38 | return self._generate_io_data(*new_args, **new_kwargs) 39 | 40 | def write_io_data(self, *args, **kwargs): 41 | new_args, new_kwargs = ProxyUtils.run( 42 | self._proxies, 43 | 'pre_write_io_data', 44 | (args, kwargs), 45 | ) 46 | return self._write_io_data(*new_args, **new_kwargs) 47 | 48 | def _generate_io_data(self, *args, **kwargs): 49 | return self.data_class(*args, **kwargs) 50 | 51 | def _write_io_data(self, *args, **kwargs): 52 | raise 53 | 54 | def _read_data(self): 55 | raise 56 | 57 | def _received_data(self, *args, **kwargs): 58 | def retrieve_io_data(): 59 | new_args, new_kwargs = ProxyUtils.run( 60 | self._proxies[::-1], 61 | 'on_retrieve_io_data', 62 | (args, kwargs), 63 | ) 64 | if new_args is not None and new_kwargs is not None: 65 | return self.data_class(*new_args, **new_kwargs) 66 | else: 67 | return None 68 | 69 | self._incoming_data_handler(retrieve_io_data) 70 | -------------------------------------------------------------------------------- /lib-python/io/proxies/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/lib-python/io/proxies/__init__.py -------------------------------------------------------------------------------- /lib-python/io/proxies/base.py: -------------------------------------------------------------------------------- 1 | class ProxyBase(object): 2 | def attach_interface(self, interface): 3 | self._interface = interface 4 | 5 | def on_retrieve_io_data(self, params): 6 | return params 7 | 8 | def pre_generate_io_data(self, params): 9 | return params 10 | 11 | def pre_write_io_data(self, params): 12 | return params 13 | -------------------------------------------------------------------------------- /lib-python/io/proxies/fragment.py: -------------------------------------------------------------------------------- 1 | from tandem.shared.io.proxies.base import ProxyBase 2 | from tandem.shared.utils.fragment import FragmentUtils 3 | 4 | 5 | class FragmentProxy(ProxyBase): 6 | def __init__(self, max_message_length=512): 7 | self._max_message_length = max_message_length 8 | 9 | def pre_generate_io_data(self, params): 10 | args, kwargs = params 11 | messages, addresses = args 12 | 13 | if type(messages) is not list: 14 | messages = [messages] 15 | 16 | new_messages = [] 17 | for message in messages: 18 | should_fragment = FragmentUtils.should_fragment( 19 | message, 20 | self._max_message_length, 21 | ) 22 | if should_fragment: 23 | new_messages.extend(FragmentUtils.fragment( 24 | message, 25 | self._max_message_length, 26 | )) 27 | else: 28 | new_messages.append(message) 29 | 30 | new_args = (new_messages, addresses) 31 | return (new_args, kwargs) 32 | 33 | def on_retrieve_io_data(self, params): 34 | args, kwargs = params 35 | if args is None or args is (None, None): 36 | return params 37 | 38 | raw_data, address = args 39 | 40 | if FragmentUtils.is_fragment(raw_data): 41 | defragmented_data = FragmentUtils.defragment(raw_data, address) 42 | if defragmented_data: 43 | new_args = (defragmented_data, address) 44 | return (new_args, kwargs) 45 | else: 46 | return (None, None) 47 | else: 48 | return params 49 | -------------------------------------------------------------------------------- /lib-python/io/proxies/list_parameters.py: -------------------------------------------------------------------------------- 1 | from tandem.shared.io.proxies.base import ProxyBase 2 | 3 | 4 | class ListParametersProxy(ProxyBase): 5 | @staticmethod 6 | def make_lists(items): 7 | new_items = [] 8 | for item in items: 9 | if type(item) is not list: 10 | item = [item] 11 | new_items.append(item) 12 | return new_items 13 | 14 | def pre_generate_io_data(self, params): 15 | args, kwargs = params 16 | return (ListParametersProxy.make_lists(args), kwargs) 17 | 18 | def pre_write_io_data(self, params): 19 | args, kwargs = params 20 | return (ListParametersProxy.make_lists(args), kwargs) 21 | -------------------------------------------------------------------------------- /lib-python/io/proxies/reliability.py: -------------------------------------------------------------------------------- 1 | from tandem.shared.io.udp_gateway import UDPGateway 2 | from tandem.shared.io.proxies.base import ProxyBase 3 | from tandem.shared.utils.reliability import ReliabilityUtils 4 | from tandem.shared.stores.reliability import ReliabilityStore 5 | import logging 6 | 7 | 8 | class ReliabilityProxy(ProxyBase): 9 | def __init__(self, time_scheduler): 10 | self._time_scheduler = time_scheduler 11 | 12 | def _handle_ack_timeout(self, ack_id, io_data): 13 | if ReliabilityUtils.should_resend_payload(ack_id): 14 | logging.info("Timeout on ack {}, resending".format(ack_id)) 15 | self._interface._write_io_data([io_data]) 16 | self._time_scheduler.run_after( 17 | ReliabilityUtils.ACK_TIMEOUT, 18 | self._handle_ack_timeout, 19 | ack_id, 20 | io_data 21 | ) 22 | 23 | def pre_write_io_data(self, params): 24 | args, kwargs = params 25 | io_datas, = args 26 | should_ack = kwargs.get('reliability', False) 27 | 28 | if not should_ack: 29 | return params 30 | 31 | new_io_datas = [] 32 | for io_data in io_datas: 33 | new_io_data = io_data 34 | new_raw_data, ack_id = ReliabilityUtils.serialize( 35 | io_data.get_data(), 36 | ) 37 | new_io_data = UDPGateway.data_class( 38 | new_raw_data, 39 | io_data.get_address(), 40 | ) 41 | 42 | ReliabilityStore.get_instance().add_payload(ack_id, new_io_data) 43 | self._time_scheduler.run_after( 44 | ReliabilityUtils.ACK_TIMEOUT, 45 | self._handle_ack_timeout, 46 | ack_id, 47 | new_io_data 48 | ) 49 | 50 | new_io_datas.append(new_io_data) 51 | 52 | new_args = (new_io_datas,) 53 | return (new_args, kwargs) 54 | 55 | def on_retrieve_io_data(self, params): 56 | args, kwargs = params 57 | raw_data, address = args 58 | 59 | if ReliabilityUtils.is_ack(raw_data): 60 | ack_id = ReliabilityUtils.parse_ack(raw_data) 61 | ReliabilityStore.get_instance().remove_payload(ack_id) 62 | return (None, None) 63 | 64 | elif ReliabilityUtils.is_ackable(raw_data): 65 | new_raw_data, ack_id = ReliabilityUtils.deserialize(raw_data) 66 | ack_payload = ReliabilityUtils.generate_ack(ack_id) 67 | self._interface.write_io_data([ 68 | self._interface.data_class(ack_payload, address), 69 | ]) 70 | 71 | new_args = new_raw_data, address 72 | return (new_args, kwargs) 73 | 74 | else: 75 | return params 76 | -------------------------------------------------------------------------------- /lib-python/io/proxies/unicode.py: -------------------------------------------------------------------------------- 1 | from tandem.shared.io.proxies.base import ProxyBase 2 | 3 | 4 | class UnicodeProxy(ProxyBase): 5 | def pre_generate_io_data(self, params): 6 | args, kwargs = params 7 | messages, addresses = args 8 | encoded_messages = [ 9 | message.encode("utf-8") if hasattr(message, "encode") else message 10 | for message in messages 11 | ] 12 | return ((encoded_messages, addresses), kwargs) 13 | 14 | def on_retrieve_io_data(self, params): 15 | args, kwargs = params 16 | if args is None: 17 | return params 18 | 19 | raw_data, address = args 20 | data = ( 21 | raw_data.decode("utf-8") if hasattr(raw_data, "decode") 22 | else raw_data 23 | ) 24 | return ((data, address), kwargs) 25 | -------------------------------------------------------------------------------- /lib-python/io/udp_gateway.py: -------------------------------------------------------------------------------- 1 | import select 2 | import socket 3 | import logging 4 | from tandem.shared.io.base import InterfaceDataBase, InterfaceBase 5 | 6 | 7 | class UDPData(InterfaceDataBase): 8 | def __init__(self, raw_data, address): 9 | super(UDPData, self).__init__(raw_data) 10 | self._address = address 11 | 12 | def get_address(self): 13 | return self._address 14 | 15 | def is_empty(self): 16 | return self._data is None and self._address is None 17 | 18 | 19 | class UDPGateway(InterfaceBase): 20 | data_class = UDPData 21 | SELECT_TIMEOUT = 0.5 22 | 23 | def __init__(self, host, port, handler_function, proxies=[]): 24 | super(UDPGateway, self).__init__(handler_function, proxies) 25 | self._host = host 26 | self._port = port 27 | self._socket = socket.socket( 28 | socket.AF_INET, 29 | socket.SOCK_DGRAM, 30 | ) 31 | self._shutdown_requested = False 32 | 33 | def start(self): 34 | self._socket.bind((self._host, self._port)) 35 | super(UDPGateway, self).start() 36 | logging.info("Tandem UDPGateway is listening on {}.".format(( 37 | self._host, 38 | self._port, 39 | ))) 40 | 41 | def stop(self): 42 | self._shutdown_requested = True 43 | # We need to ensure the reader thread has been joined before closing 44 | # the socket to make sure we don't call select() on an invalid file 45 | # descriptor. 46 | super(UDPGateway, self).stop() 47 | self._socket.close() 48 | 49 | def get_port(self): 50 | return self._socket.getsockname()[1] 51 | 52 | def _generate_io_data(self, *args, **kwargs): 53 | messages, addresses = args 54 | 55 | data = [] 56 | for address in addresses: 57 | for message in messages: 58 | data.append(UDPData(message, address)) 59 | 60 | return data 61 | 62 | def _write_io_data(self, *args, **kwargs): 63 | io_datas, = args 64 | 65 | for io_data in io_datas: 66 | message = io_data.get_data() 67 | address = io_data.get_address() 68 | bytes_sent = 0 69 | 70 | while bytes_sent < len(message): 71 | bytes_sent += self._socket.sendto( 72 | message[bytes_sent:], 73 | address 74 | ) 75 | 76 | def _read_data(self): 77 | while not self._shutdown_requested: 78 | ready_to_read, _, _ = select.select( 79 | [self._socket], 80 | [], 81 | [], 82 | UDPGateway.SELECT_TIMEOUT, 83 | ) 84 | if len(ready_to_read) == 0: 85 | # If no descriptors are ready to read, it means the select() 86 | # call timed out. So check if we should exit and, if not, wait 87 | # for data again. 88 | continue 89 | 90 | raw_data, address = self._socket.recvfrom(4096) 91 | logging.debug("Received data from {}:{}.".format(*address)) 92 | self._received_data(raw_data, address) 93 | 94 | logging.info( 95 | "Tandem has closed the UDP gateway on port {}." 96 | .format(self._port), 97 | ) 98 | -------------------------------------------------------------------------------- /lib-python/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/lib-python/models/__init__.py -------------------------------------------------------------------------------- /lib-python/models/base.py: -------------------------------------------------------------------------------- 1 | class ModelBase(object): 2 | pass 3 | -------------------------------------------------------------------------------- /lib-python/models/fragment.py: -------------------------------------------------------------------------------- 1 | from tandem.shared.models.base import ModelBase 2 | 3 | 4 | class Fragment(ModelBase): 5 | def __init__( 6 | self, 7 | total_fragments, 8 | fragment_number, 9 | payload, 10 | ): 11 | self._total_fragments = total_fragments 12 | self._fragment_number = fragment_number 13 | self._payload = payload 14 | 15 | def get_total_fragments(self): 16 | return self._total_fragments 17 | 18 | def get_fragment_number(self): 19 | return self._fragment_number 20 | 21 | def get_payload(self): 22 | return self._payload 23 | 24 | 25 | class FragmentGroup(ModelBase): 26 | def __init__(self, total_fragments): 27 | self._total_fragments = total_fragments 28 | self._buffer = [None for _ in range(total_fragments)] 29 | 30 | def add_fragment(self, fragment): 31 | fragment_number = fragment.get_fragment_number() 32 | if self._buffer[fragment_number] is None: 33 | self._buffer[fragment_number] = fragment.get_payload() 34 | 35 | def is_complete(self): 36 | non_empty_fragments = list(filter(lambda x: x, self._buffer)) 37 | return len(non_empty_fragments) >= self._total_fragments 38 | 39 | def defragment(self): 40 | return b"".join(self._buffer) if self.is_complete() else None 41 | -------------------------------------------------------------------------------- /lib-python/models/peer.py: -------------------------------------------------------------------------------- 1 | from tandem.shared.models.base import ModelBase 2 | 3 | 4 | class Peer(ModelBase): 5 | def __init__(self, id, public_address, private_address=None): 6 | self._id = id 7 | self._public_address = public_address 8 | self._private_address = private_address 9 | 10 | def __eq__(self, other): 11 | return ( 12 | self._id == other._id and 13 | self._public_address == other._public_address and 14 | self._private_address == other._private_address 15 | ) 16 | 17 | def get_id(self): 18 | return self._id 19 | 20 | def get_addresses(self): 21 | addresses = [self._public_address] 22 | if self._private_address is not None: 23 | addresses.append(self._private_address) 24 | return addresses 25 | 26 | def get_public_address(self): 27 | return self._public_address 28 | 29 | def get_private_address(self): 30 | return self._private_address 31 | -------------------------------------------------------------------------------- /lib-python/protocol/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/lib-python/protocol/__init__.py -------------------------------------------------------------------------------- /lib-python/protocol/handlers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/lib-python/protocol/handlers/__init__.py -------------------------------------------------------------------------------- /lib-python/protocol/handlers/addressed.py: -------------------------------------------------------------------------------- 1 | from tandem.shared.protocol.handlers.base import ProtocolHandlerBase 2 | 3 | 4 | class AddressedHandler(ProtocolHandlerBase): 5 | def _extra_handler_arguments(self, io_data): 6 | return [io_data.get_address()] 7 | -------------------------------------------------------------------------------- /lib-python/protocol/handlers/base.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import json 3 | from tandem.shared.protocol.messages.base import ProtocolMarshalError 4 | 5 | 6 | class ProtocolHandlerBase(object): 7 | def _protocol_message_utils(self): 8 | return None 9 | 10 | def _protocol_message_handlers(self): 11 | return None 12 | 13 | def _extra_handler_arguments(self, io_data): 14 | return [] 15 | 16 | def handle_raw_data(self, retrieve_io_data): 17 | try: 18 | io_data = retrieve_io_data() 19 | if io_data is None or io_data.is_empty(): 20 | return 21 | 22 | data_as_dict = json.loads(io_data.get_data()) 23 | handled = self.handle_message(data_as_dict, io_data) 24 | 25 | if not handled: 26 | logging.info( 27 | "Protocol message was not handled because " 28 | "no handler was registered.", 29 | ) 30 | 31 | except json.JSONDecodeError: 32 | logging.info( 33 | "Protocol message was ignored because it was not valid JSON.", 34 | ) 35 | 36 | except: 37 | logging.exception("Exception when handling protocol message:") 38 | raise 39 | 40 | def handle_message(self, data_as_dict, io_data): 41 | try: 42 | message = \ 43 | self._protocol_message_utils().deserialize(data_as_dict) 44 | items = self._protocol_message_handlers().items() 45 | 46 | for message_type, handler in items: 47 | if message_type == message.type.value: 48 | handler(message, *self._extra_handler_arguments(io_data)) 49 | return True 50 | 51 | return False 52 | 53 | except ProtocolMarshalError: 54 | return False 55 | -------------------------------------------------------------------------------- /lib-python/protocol/handlers/multi.py: -------------------------------------------------------------------------------- 1 | from tandem.shared.protocol.handlers.base import ProtocolHandlerBase 2 | 3 | 4 | class MultiProtocolHandler(ProtocolHandlerBase): 5 | def __init__(self, *handlers): 6 | self._handlers = [handler for handler in handlers] 7 | 8 | def handle_message(self, data_as_dict, io_data): 9 | for handler in self._handlers: 10 | handled = handler.handle_message(data_as_dict, io_data) 11 | if handled: 12 | return True 13 | 14 | return False 15 | -------------------------------------------------------------------------------- /lib-python/protocol/messages/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/lib-python/protocol/messages/__init__.py -------------------------------------------------------------------------------- /lib-python/protocol/messages/base.py: -------------------------------------------------------------------------------- 1 | import json 2 | import enum 3 | 4 | 5 | class ProtocolMarshalError(ValueError): 6 | pass 7 | 8 | 9 | class ProtocolMessageTypeBase(enum.Enum): 10 | pass 11 | 12 | 13 | class ProtocolMessageBase(object): 14 | def __init__(self, message_type, **kwargs): 15 | for key in self._payload_keys(): 16 | setattr(self, key, kwargs.get(key, None)) 17 | 18 | self.type = message_type 19 | 20 | def _payload_keys(self): 21 | return None 22 | 23 | def to_payload(self): 24 | return {key: getattr(self, key, None) for key in self._payload_keys()} 25 | 26 | @classmethod 27 | def from_payload(cls, **kwargs): 28 | return cls(**kwargs) 29 | 30 | 31 | class ProtocolUtilsBase(object): 32 | @classmethod 33 | def _protocol_message_constructors(cls): 34 | return None 35 | 36 | @staticmethod 37 | def serialize(message): 38 | as_dict = { 39 | "type": message.type.value, 40 | "payload": message.to_payload(), 41 | "version": 1, 42 | } 43 | return json.dumps(as_dict) 44 | 45 | @classmethod 46 | def deserialize(cls, as_dict): 47 | data_message_type = as_dict["type"] 48 | data_payload = as_dict["payload"] 49 | items = cls._protocol_message_constructors().items() 50 | 51 | for message_type, target_class in items: 52 | if message_type == data_message_type: 53 | return target_class.from_payload(**data_payload) 54 | 55 | raise ProtocolMarshalError 56 | -------------------------------------------------------------------------------- /lib-python/protocol/messages/rendezvous.py: -------------------------------------------------------------------------------- 1 | from tandem.shared.protocol.messages.base import ( 2 | ProtocolMessageBase, 3 | ProtocolMessageTypeBase, 4 | ProtocolUtilsBase, 5 | ) 6 | from tandem.shared.utils.static_value import static_value as staticvalue 7 | 8 | 9 | class RendezvousProtocolMessageType(ProtocolMessageTypeBase): 10 | ConnectRequest = "rv-connect-request" 11 | SetupParameters = "rv-setup-parameters" 12 | Error = "rv-error" 13 | 14 | 15 | class ConnectRequest(ProtocolMessageBase): 16 | """ 17 | Sent by an agent to request to join an existing session. 18 | """ 19 | def __init__(self, **kwargs): 20 | super(ConnectRequest, self).__init__( 21 | RendezvousProtocolMessageType.ConnectRequest, 22 | **kwargs, 23 | ) 24 | 25 | @staticvalue 26 | def _payload_keys(self): 27 | return ["session_id", "my_id", "private_address"] 28 | 29 | 30 | class SetupParameters(ProtocolMessageBase): 31 | """ 32 | Sent by the server to agents to inform them to connect. 33 | """ 34 | def __init__(self, **kwargs): 35 | super(SetupParameters, self).__init__( 36 | RendezvousProtocolMessageType.SetupParameters, 37 | **kwargs, 38 | ) 39 | 40 | @staticvalue 41 | def _payload_keys(self): 42 | return ["session_id", "peer_id", "initiate", "public", "private"] 43 | 44 | 45 | class Error(ProtocolMessageBase): 46 | """ 47 | Sent by the server to send an error message. 48 | """ 49 | def __init__(self, **kwargs): 50 | super(Error, self).__init__( 51 | RendezvousProtocolMessageType.Error, 52 | **kwargs, 53 | ) 54 | 55 | @staticvalue 56 | def _payload_keys(self): 57 | return ["message"] 58 | 59 | 60 | class RendezvousProtocolUtils(ProtocolUtilsBase): 61 | @classmethod 62 | @staticvalue 63 | def _protocol_message_constructors(cls): 64 | return { 65 | RendezvousProtocolMessageType.ConnectRequest.value: 66 | ConnectRequest, 67 | RendezvousProtocolMessageType.SetupParameters.value: 68 | SetupParameters, 69 | RendezvousProtocolMessageType.Error.value: Error, 70 | } 71 | -------------------------------------------------------------------------------- /lib-python/stores/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/lib-python/stores/__init__.py -------------------------------------------------------------------------------- /lib-python/stores/base.py: -------------------------------------------------------------------------------- 1 | class StoreBase(object): 2 | instance = None 3 | 4 | @classmethod 5 | def get_instance(cls): 6 | if not cls.instance: 7 | cls.instance = cls() 8 | return cls.instance 9 | 10 | @classmethod 11 | def reset_instance(cls): 12 | cls.instance = None 13 | -------------------------------------------------------------------------------- /lib-python/stores/fragment.py: -------------------------------------------------------------------------------- 1 | from tandem.shared.models.fragment import FragmentGroup 2 | from tandem.shared.stores.base import StoreBase 3 | 4 | 5 | class FragmentStore(StoreBase): 6 | def __init__(self): 7 | self._peer_fragment_groups = {} 8 | 9 | def insert_fragment(self, address, message_id, fragment): 10 | if address not in self._peer_fragment_groups: 11 | self._peer_fragment_groups[address] = {} 12 | 13 | fragment_groups = self._peer_fragment_groups[address] 14 | if message_id not in fragment_groups: 15 | new_group = FragmentGroup(fragment.get_total_fragments()) 16 | fragment_groups[message_id] = new_group 17 | 18 | fragment_groups[message_id].add_fragment(fragment) 19 | 20 | def get_fragment_group(self, address, message_id): 21 | return self._peer_fragment_groups[address][message_id] 22 | 23 | def remove_fragment_group(self, address, message_id): 24 | del self._peer_fragment_groups[address][message_id] 25 | -------------------------------------------------------------------------------- /lib-python/stores/reliability.py: -------------------------------------------------------------------------------- 1 | from tandem.shared.stores.base import StoreBase 2 | 3 | 4 | class ReliabilityStore(StoreBase): 5 | def __init__(self): 6 | self._payloads = {} 7 | 8 | def add_payload(self, payload_id, payload): 9 | self._payloads[payload_id] = payload 10 | 11 | def get_payload(self, payload_id): 12 | return self._payloads.get(payload_id, None) 13 | 14 | def remove_payload(self, payload_id): 15 | if payload_id in self._payloads: 16 | del self._payloads[payload_id] 17 | -------------------------------------------------------------------------------- /lib-python/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/lib-python/utils/__init__.py -------------------------------------------------------------------------------- /lib-python/utils/fragment.py: -------------------------------------------------------------------------------- 1 | from tandem.shared.stores.fragment import FragmentStore 2 | from tandem.shared.models.fragment import Fragment 3 | 4 | 5 | class FragmentUtils(object): 6 | HEADER = b"\x54\x01" 7 | FRAGMENT_HEADER = b"\x46\x52" 8 | FRAGMENT_HEADER_LENGTH = len(HEADER) + len(FRAGMENT_HEADER) + 6 9 | 10 | MAX_SEQUENCE_NUMBER = int(0xFFFF) 11 | next_sequence_number = -1 12 | 13 | @classmethod 14 | def is_fragment(cls, message): 15 | return ( 16 | message[0:2] == cls.HEADER and 17 | message[2:4] == cls.FRAGMENT_HEADER 18 | ) 19 | 20 | @staticmethod 21 | def should_fragment(message, max_message_length): 22 | return len(message) > max_message_length 23 | 24 | @classmethod 25 | def get_next_sequence_number(cls): 26 | cls.next_sequence_number += 1 27 | cls.next_sequence_number %= cls.MAX_SEQUENCE_NUMBER + 1 28 | 29 | return cls.next_sequence_number 30 | 31 | @staticmethod 32 | def serialize(fragment, sequence_number): 33 | result = [] 34 | result.append(FragmentUtils.HEADER) 35 | result.append(FragmentUtils.FRAGMENT_HEADER) 36 | result.append( 37 | fragment.get_total_fragments().to_bytes(2, byteorder="big") 38 | ) 39 | result.append( 40 | sequence_number.to_bytes(2, byteorder="big") 41 | ) 42 | result.append( 43 | fragment.get_fragment_number().to_bytes(2, byteorder="big") 44 | ) 45 | result.append(fragment.get_payload()) 46 | return b"".join(result) 47 | 48 | @staticmethod 49 | def deserialize(message): 50 | total_fragments = int.from_bytes(message[4:6], byteorder="big") 51 | sequence_number = int.from_bytes(message[6:8], byteorder="big") 52 | fragment_number = int.from_bytes(message[8:10], byteorder="big") 53 | payload = message[10:] 54 | 55 | new_fragment = Fragment(total_fragments, fragment_number, payload) 56 | return new_fragment, sequence_number 57 | 58 | @classmethod 59 | def fragment(cls, payload, max_message_length): 60 | max_payload_length = max_message_length - cls.FRAGMENT_HEADER_LENGTH 61 | 62 | payloads = [ 63 | payload[i:i + max_payload_length] 64 | for i in range(0, len(payload), max_payload_length) 65 | ] 66 | 67 | fragments = [ 68 | Fragment(len(payloads), index, payload) 69 | for index, payload in enumerate(payloads) 70 | ] 71 | 72 | sequence_number = FragmentUtils.get_next_sequence_number() 73 | messages = [ 74 | FragmentUtils.serialize(fragment, sequence_number) 75 | for fragment in fragments 76 | ] 77 | 78 | return messages 79 | 80 | @staticmethod 81 | def defragment(raw_data, sender_address): 82 | fragment_store = FragmentStore.get_instance() 83 | fragment, sequence_number = FragmentUtils.deserialize(raw_data) 84 | fragment_store.insert_fragment( 85 | sender_address, 86 | sequence_number, 87 | fragment 88 | ) 89 | fragment_group = fragment_store.get_fragment_group( 90 | sender_address, 91 | sequence_number, 92 | ) 93 | 94 | defragmented_data = fragment_group.defragment() 95 | if fragment_group.is_complete(): 96 | fragment_store.remove_fragment_group( 97 | sender_address, 98 | sequence_number, 99 | ) 100 | return defragmented_data 101 | -------------------------------------------------------------------------------- /lib-python/utils/proxy.py: -------------------------------------------------------------------------------- 1 | class ProxyUtils(object): 2 | @staticmethod 3 | def run(proxies, method, data): 4 | for proxy in proxies: 5 | data = getattr(proxy, method, lambda x: x)(data) 6 | 7 | return data 8 | -------------------------------------------------------------------------------- /lib-python/utils/relay.py: -------------------------------------------------------------------------------- 1 | class RelayUtils(object): 2 | HEADER = b"\x54\x01" 3 | RELAY_HEADER = b"\x52\x45" 4 | 5 | @classmethod 6 | def is_relay(cls, raw_data): 7 | return ( 8 | raw_data[0:2] == cls.HEADER and 9 | raw_data[2:4] == cls.RELAY_HEADER 10 | ) 11 | 12 | @staticmethod 13 | def serialize(payload, address): 14 | result = [] 15 | ip, port = address 16 | ip_binary = map( 17 | lambda x: (int(x)).to_bytes(1, byteorder="big"), 18 | ip.split("."), 19 | ) 20 | 21 | result.append(RelayUtils.HEADER) 22 | result.append(RelayUtils.RELAY_HEADER) 23 | result.extend(ip_binary) 24 | result.append(port.to_bytes(2, byteorder="big")) 25 | result.append(payload) 26 | 27 | return b"".join(result) 28 | 29 | @staticmethod 30 | def deserialize(raw_data): 31 | ip = ".".join([ 32 | str(int.from_bytes(raw_data[4:5], byteorder="big")), 33 | str(int.from_bytes(raw_data[5:6], byteorder="big")), 34 | str(int.from_bytes(raw_data[6:7], byteorder="big")), 35 | str(int.from_bytes(raw_data[7:8], byteorder="big")), 36 | ]) 37 | port = int.from_bytes(raw_data[8:10], byteorder="big") 38 | 39 | address = (ip, port) 40 | payload = raw_data[10:] 41 | 42 | return payload, address 43 | -------------------------------------------------------------------------------- /lib-python/utils/reliability.py: -------------------------------------------------------------------------------- 1 | from tandem.shared.stores.reliability import ReliabilityStore 2 | 3 | 4 | class ReliabilityUtils(object): 5 | HEADER = b"\x54\x01" 6 | RELIABILITY_HEADER = b"\x52\x4C" 7 | ACK_HEADER = b"\x41\x43" 8 | ACK_TIMEOUT = 3 9 | 10 | MAX_ACK_NUMBER = int(0xFFFF) 11 | next_ack_number = -1 12 | 13 | @classmethod 14 | def get_next_ack_number(cls): 15 | cls.next_ack_number += 1 16 | cls.next_ack_number %= cls.MAX_ACK_NUMBER + 1 17 | 18 | return cls.next_ack_number 19 | 20 | @classmethod 21 | def is_ack(cls, raw_data): 22 | return ( 23 | raw_data[0:2] == cls.HEADER and 24 | raw_data[2:4] == cls.ACK_HEADER 25 | ) 26 | 27 | @classmethod 28 | def is_ackable(cls, raw_data): 29 | return ( 30 | raw_data[0:2] == cls.HEADER and 31 | raw_data[2:4] == cls.RELIABILITY_HEADER 32 | ) 33 | 34 | @staticmethod 35 | def should_resend_payload(ack_id): 36 | return ReliabilityStore.get_instance().get_payload(ack_id) 37 | 38 | @staticmethod 39 | def generate_ack(ack_id): 40 | result = [] 41 | result.append(ReliabilityUtils.HEADER) 42 | result.append(ReliabilityUtils.ACK_HEADER) 43 | result.append((ack_id).to_bytes(2, byteorder="big")) 44 | return b"".join(result) 45 | 46 | @staticmethod 47 | def parse_ack(raw_data): 48 | return int.from_bytes(raw_data[4:6], byteorder="big") 49 | 50 | @staticmethod 51 | def serialize(payload): 52 | result = [] 53 | ack_number = ReliabilityUtils.get_next_ack_number() 54 | result.append(ReliabilityUtils.HEADER) 55 | result.append(ReliabilityUtils.RELIABILITY_HEADER) 56 | result.append(ack_number.to_bytes(2, byteorder="big")) 57 | result.append(payload) 58 | return b"".join(result), ack_number 59 | 60 | @staticmethod 61 | def deserialize(raw_data): 62 | ack_id = int.from_bytes(raw_data[4:6], byteorder="big") 63 | payload = raw_data[6:] 64 | return payload, ack_id 65 | -------------------------------------------------------------------------------- /lib-python/utils/static_value.py: -------------------------------------------------------------------------------- 1 | def static_value(inner_function): 2 | # Using dictionary to workaround Python2's lack of `nonlocal` 3 | dict_value = {} 4 | 5 | def outer_function(*args, **kwargs): 6 | if dict_value.get('value', None) is None: 7 | dict_value['value'] = inner_function(*args, **kwargs) 8 | 9 | return dict_value['value'] 10 | 11 | return outer_function 12 | -------------------------------------------------------------------------------- /lib-python/utils/time_scheduler.py: -------------------------------------------------------------------------------- 1 | import sched 2 | import time 3 | from threading import Thread, Event 4 | 5 | 6 | class TimeScheduler: 7 | """ 8 | Schedules tasks to run in the future on an executor. 9 | 10 | The queue of tasks to execute is inspected every 11 | resolution_seconds. So the minimal delay for a task 12 | is the resolution of this scheduler. 13 | """ 14 | def __init__(self, executor, resolution_seconds=0.1): 15 | self._executor = executor 16 | self._resolution_seconds = resolution_seconds 17 | 18 | self._shutting_down = False 19 | self._shut_down_event = Event() 20 | self._runner = Thread(target=self._run_scheduler) 21 | self._scheduler = sched.scheduler(time.time, time.sleep) 22 | 23 | def run_after(self, delay_seconds, function, *args, **kwargs): 24 | """ 25 | Schedules the specified function on the executor after delay_seconds. 26 | 27 | This returns a handle that has a cancel() method to cancel the 28 | request to run this function. 29 | """ 30 | handle = _Handle(self) 31 | handle.set_event_handle(self._schedule_after( 32 | delay_seconds, 33 | function, 34 | handle, 35 | lambda: None, 36 | *args, 37 | **kwargs, 38 | )) 39 | return handle 40 | 41 | def run_every(self, interval_seconds, function, *args, **kwargs): 42 | """ 43 | Schedules the specified function at least every interval_seconds. 44 | 45 | This returns a handle that has a cancel() method to cancel the 46 | request to run this function. 47 | 48 | This only guarantees that at least interval_seconds elapses 49 | between each invocation of the function. It does not guarantee 50 | that the function runs exactly every interval_seconds. 51 | """ 52 | handle = _Handle(self) 53 | 54 | def reschedule(): 55 | handle.set_event_handle(self._schedule_after( 56 | interval_seconds, 57 | function, 58 | handle, 59 | reschedule, 60 | *args, 61 | **kwargs, 62 | )) 63 | 64 | reschedule() 65 | return handle 66 | 67 | def start(self): 68 | """ 69 | Starts this scheduler. 70 | 71 | Until this is called, no tasks will actually be scheduled. 72 | However the scheduler will still accept schedule requests. 73 | """ 74 | self._runner.start() 75 | 76 | def stop(self): 77 | """ 78 | Stops this scheduler. 79 | 80 | All remaining pending tasks after this returns will no 81 | longer be scheduled. This does not wait for all pending 82 | tasks to be scheduled. 83 | """ 84 | self._shutting_down = True 85 | self._shut_down_event.set() 86 | self._runner.join() 87 | 88 | def _cancel(self, event_handle): 89 | try: 90 | self._scheduler.cancel(event_handle) 91 | except ValueError: 92 | pass 93 | 94 | def _schedule_after( 95 | self, 96 | delay_seconds, 97 | function, 98 | handle, 99 | epilogue, 100 | *args, 101 | **kwargs, 102 | ): 103 | return self._scheduler.enter( 104 | delay_seconds, 105 | 0, 106 | self._executor.submit, 107 | (self._run_if_not_cancelled, function, handle, epilogue, *args), 108 | kwargs, 109 | ) 110 | 111 | def _run_if_not_cancelled( 112 | self, 113 | function, 114 | handle, 115 | epilogue, 116 | *args, 117 | **kwargs, 118 | ): 119 | if handle.is_cancelled(): 120 | return 121 | try: 122 | function(*args, **kwargs) 123 | finally: 124 | epilogue() 125 | 126 | def _run_scheduler(self): 127 | while not self._shutting_down: 128 | self._scheduler.run(blocking=False) 129 | self._shut_down_event.wait(timeout=self._resolution_seconds) 130 | 131 | 132 | class _Handle: 133 | def __init__(self, scheduler): 134 | self._scheduler = scheduler 135 | self._event_handle = None 136 | self._cancelled = False 137 | 138 | def cancel(self): 139 | self._cancelled = True 140 | self._scheduler._cancel(self._event_handle) 141 | 142 | def is_cancelled(self): 143 | return self._cancelled 144 | 145 | def set_event_handle(self, new_handle): 146 | self._event_handle = new_handle 147 | -------------------------------------------------------------------------------- /plugins/sublime/Main.sublime-menu: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "caption": "Tandem", 4 | "mnemonic": "T", 5 | "id": "tandem", 6 | "children": 7 | [ 8 | { "caption": "Start Session", "mnemonic": "S", "command": "tandem", "args": { "show_gui": "True" } }, 9 | { "caption": "Join Existing Session", "mnemonic": "J", "command": "tandem_connect" }, 10 | { "caption": "Show Session ID", "mnemonic": "S", "command": "tandem_session", "args": { "show_gui": "True" } }, 11 | { "caption": "Leave Session", "mnemonic": "L", "command": "tandem_stop", "args": { "show_gui": "True" } } 12 | ] 13 | } 14 | ] 15 | -------------------------------------------------------------------------------- /plugins/sublime/README.md: -------------------------------------------------------------------------------- 1 | # Tandem 2 | 3 | Tandem is an add-on for your favorite text editor that enables peer-to-peer collaborative editing across different editors. 4 | 5 | This repository contains code for the Sublime Text 3 plugin. For more details on Tandem, visit [our website](http://typeintandem.com), or our [mono-repository containing all other source code.](https://github.com/typeintandem/tandem) 6 | 7 | ## Installation 8 | To install, you must have a copy of Sublime Text 3 installed. Older versions of Sublime are not supported. 9 | You must also have `python3` and `node.js` installed. 10 | 11 | Sublime Text 3 users have the option of installing in one of three ways: 12 | - Using Package Control, searching for Tandem in the global package list. 13 | This is the simplest and easiest way to install Tandem. 14 | *Note: This option doesn't work yet as we're [waiting to be accepted into the 15 | official package control 16 | repository](https://github.com/wbond/package_control_channel/pull/6986)* 17 | - **[Recommended]** Using Package Control, adding this repository as a source. You will then need to install the Tandem package from this repository. Due to name conflicts, you will need to navigate to the default package installation directory (e.g. `~/Library/Application\ Support/Sublime\ Text\ 3/Packages`) and rename the directory from `sublime` to `tandem`. 18 | Installing from the official source will remove the need to do this. 19 | - Installing manually. Clone the repository to the Sublime Text 3 packages directory, and place it in a folder called `tandem`. 20 | 21 | ## Usage 22 | Tandem users can choose either start a collaborative session or join an existing one. 23 | 24 | - To start your session, select `Tandem > Start Session` from the menu or the command palette. This will create a session and give you a session ID you can share with participants you’d like to invite. Content in your buffer will be shared, so open a new view before creating a session if you don’t wish to share the contents. 25 | - To join an existing session, select `Tandem > Join Existing Session` from the menu or command palette. Enter the session ID given to you, press enter, and begin collaborating! Your session will be opened in a new view. 26 | - Any user can leave a session at any time - all other peers can continue working in the session. Simply use `Tandem > Leave Session` from the menu or command palette. 27 | - If you want to find your session ID again, select `Tandem > Show Session ID` to view your session ID. 28 | 29 | **Note: You MUST leave an active session before exiting Sublime Text! Due to editor limitations, failure to do so will cause the networking agent to remain alive and consume system resources.** 30 | 31 | ### Advanced Usage 32 | While commands have GUI hooks (menu option, command palette shortcut), advanced users can also start, join and leave sessions using commands. 33 | 34 | Open the command palette (Ctrl + `). From there, use one of the following commands: 35 | - `view.run_command("tandem")` : starts a session and prints the session ID 36 | - `view.run_command("tandem_connect", "abcd-1234")`: joins a tandem session with the specified session ID (in this case, "abcd-1234") 37 | - `view.run_command("tandem_stop")`: leaves an existing Tandem session 38 | - `view.run_command("tandem_session")`: displays the current session ID 39 | 40 | ## Terms of Service 41 | By using Tandem, you agree that any modified versions of Tandem will not use 42 | the rendezvous server hosted by the owners. You must host and use your own copy 43 | of the rendezvous server. We want to provide a good user experience for Tandem, 44 | and it would be difficult to do that with modified clients as well. 45 | 46 | You can launch the rendezvous server by running `python3 ./rendezvous/main.py`. 47 | Change the address of the rendezvous server used by the agent in the 48 | configuration file to point to your server's host. This file is located at: 49 | `agent/tandem/agent/configuration.py` 50 | 51 | ## License 52 | Copyright (c) 2018 Team Lightly 53 | 54 | See [LICENSE.txt](LICENSE.txt) 55 | 56 | Licensed under the Apache License, Version 2.0 (the "License"); 57 | you may not use this file except in compliance with the License. 58 | You may obtain a copy of the License at: 59 | 60 | http://www.apache.org/licenses/LICENSE-2.0 61 | 62 | ## Authors 63 | Team Lightly 64 | [Geoffrey Yu](https://github.com/geoffxy), [Jamiboy 65 | Mohammad](https://github.com/jamiboym) and [Sameer 66 | Chitley](https://github.com/rageandqq) 67 | 68 | We are a team of senior Software Engineering students at the University of 69 | Waterloo. 70 | Tandem was created as our [Engineering Capstone Design 71 | Project](https://uwaterloo.ca/capstone-design). 72 | -------------------------------------------------------------------------------- /plugins/sublime/__init__.py: -------------------------------------------------------------------------------- 1 | from sublime_plugin import WindowCommand, TextCommand 2 | import sublime 3 | 4 | __all__ = ['ST2', 'ST3', 'WindowAndTextCommand', 'Settings', 'FileSettings'] 5 | 6 | ST2 = sublime.version().startswith('2') 7 | ST3 = not ST2 8 | 9 | 10 | class WindowAndTextCommand(WindowCommand, TextCommand): 11 | """A class to derive from when using a Window- and a TextCommand in one 12 | class (e.g. when you make a build system that should/could also be called 13 | from the command palette with the view in its focus). 14 | 15 | Defines both self.view and self.window. 16 | 17 | Be careful that self.window may be ``None`` when called as a 18 | TextCommand because ``view.window()`` is not really safe and will 19 | fail in quite a few cases. Since the compromise of using 20 | ``sublime.active_window()`` in that case is not wanted by every 21 | command I refused from doing so. Thus, the command's on duty to check 22 | whether the window is valid. 23 | 24 | Since this class derives from both Window- and a TextCommand it is also 25 | callable with the known methods, like 26 | ``window.run_command("window_and_text")``. 27 | I defined a dummy ``run`` method to prevent parameters from raising an 28 | exception so this command call does exactly nothing. 29 | Still a better method than having the parent class (the command you 30 | will define) derive from three classes with the limitation that this 31 | class must be the first one (the *Command classes do not use super() 32 | for multi-inheritance support; neither do I but apparently I have 33 | reasons). 34 | """ 35 | def __init__(self, param): 36 | # no super() call! this would get the references confused 37 | if isinstance(param, sublime.Window): 38 | self.window = param 39 | self._window_command = True # probably called from build system 40 | self.typ = WindowCommand 41 | elif isinstance(param, sublime.View): 42 | self.view = param 43 | self._window_command = False 44 | self.typ = TextCommand 45 | else: 46 | raise TypeError("Something really bad happened and you are responsible") 47 | 48 | self._update_members() 49 | 50 | def _update_members(self): 51 | if self._window_command: 52 | self.view = self.window.active_view() 53 | else: 54 | self.window = self.view.window() 55 | 56 | def run_(self, *args): 57 | """Wraps the other run_ method implementations from sublime_plugin. 58 | Required to update the self.view and self.window variables. 59 | """ 60 | self._update_members() 61 | # Obviously `super` does not work here 62 | self.typ.run_(self, *args) 63 | 64 | 65 | class Settings(object): 66 | """Helper class for accessing sublime.Settings' values. 67 | 68 | Settings(settings, none_erases=False) 69 | 70 | * settings (sublime.Settings) 71 | Should be self-explanatory. 72 | 73 | * none_erases (bool, optional) 74 | Iff ``True`` a setting's key will be erased when setting it to 75 | ``None``. This only has a meaning when the key you erase is 76 | defined in a parent Settings collection which would be 77 | retrieved in that case. 78 | 79 | Defines the default methods for sublime.Settings: 80 | 81 | get(key, default=None) 82 | set(key, value) 83 | erase(key) 84 | has(key) 85 | add_on_change(key, on_change) 86 | clear_on_change(key, on_change) 87 | 88 | http://www.sublimetext.com/docs/2/api_reference.html#sublime.Settings 89 | 90 | If ``none_erases == True`` you can erase a key when setting it to 91 | ``None``. This only has a meaning when the key you erase is defined in 92 | a parent Settings collection which would be retrieved in that case. 93 | 94 | The following methods can be used to retrieve a setting's value: 95 | 96 | value = self.get('key', default) 97 | value = self['key'] 98 | value = self.key_without_spaces 99 | 100 | The following methods can be used to set a setting's value: 101 | 102 | self.set('key', value) 103 | self['key'] = value 104 | self.key_without_spaces = value 105 | 106 | The following methods can be used to erase a key in the setting: 107 | 108 | self.erase('key') 109 | self.set('key', None) or similar # iff ``none_erases == True`` 110 | del self.key_without_spaces 111 | 112 | ! Important: 113 | Don't use the attribute method with one of these keys; ``dir(Settings)``: 114 | 115 | ['__class__', '__delattr__', '__dict__', '__doc__', '__format__', 116 | '__getattr__', '__getattribute__', '__getitem__', '__hash__', 117 | '__init__', '__module__', '__new__', '__reduce__', '__reduce_ex__', 118 | '__repr__', '__setattr__', '__setitem__', '__sizeof__', '__str__', 119 | '__subclasshook__', '__weakref__', 120 | 121 | '_none_erases', '_s', '_settable_attributes', 122 | 123 | 'add_on_change', 'clear_on_change', 124 | 'erase', 'get', 'has', 'set'] 125 | 126 | Getting will return the respective function/value, setting will do 127 | nothing. Setting of _leading_underline_values from above will result in 128 | unpredictable behavior. Please don't do this! And re-consider even when 129 | you know what you're doing. 130 | """ 131 | _none_erases = False 132 | _s = None 133 | _settable_attributes = ('_s', '_none_erases') # allow only setting of these attributes 134 | 135 | def __init__(self, settings, none_erases=False): 136 | if not isinstance(settings, sublime.Settings): 137 | raise ValueError("Not an instance of sublime.Settings") 138 | self._s = settings 139 | self._none_erases = none_erases 140 | 141 | def get(self, key, default=None): 142 | """Returns the named setting, or ``default`` if it's not defined. 143 | """ 144 | return self._s.get(key, default) 145 | 146 | def set(self, key, value): 147 | """Sets the named setting. Only primitive types, lists, and 148 | dictionaries are accepted. 149 | Erases the key iff ``value is None``. 150 | """ 151 | if value is None and self._none_erases: 152 | self.erase(key) 153 | else: 154 | self._s.set(key, value) 155 | 156 | def erase(self, key): 157 | """Removes the named setting. Does not remove it from any parent Settings. 158 | """ 159 | self._s.erase(key) 160 | 161 | def has(self, key): 162 | """Returns true iff the named option exists in this set of Settings or 163 | one of its parents. 164 | """ 165 | return self._s.has(key) 166 | 167 | def add_on_change(self, key, on_change): 168 | """Register a callback to be run whenever the setting with this key in 169 | this object is changed. 170 | """ 171 | self._s.add_on_change(key, on_change) 172 | 173 | def clear_on_change(self, key, on_change): 174 | """Remove all callbacks registered with the given key. 175 | """ 176 | self._s.clear_on_change(key, on_change) 177 | 178 | def __getitem__(self, key): 179 | """self[key]""" 180 | return self.get(key) 181 | 182 | def __setitem__(self, key, value): 183 | """self[key] = value""" 184 | self.set(key, value) 185 | 186 | def __getattr__(self, key): 187 | """self.key_without_spaces""" 188 | return self.get(key) 189 | 190 | def __setattr__(self, key, value): 191 | """self.key_without_spaces = value""" 192 | if key in self._settable_attributes: 193 | object.__setattr__(self, key, value) 194 | else: 195 | self.set(key, value) 196 | 197 | def __delattr__(self, key): 198 | """del self.key_without_spaces""" 199 | if key in dir(self): 200 | return 201 | else: 202 | self.erase(key) 203 | 204 | 205 | class FileSettings(Settings): 206 | """Helper class for accessing sublime.Settings' values. 207 | 208 | Derived from sublime_lib.Settings. Please also read the documentation 209 | there. 210 | 211 | FileSettings(name, none_erases=False) 212 | 213 | * name (str) 214 | The file name that's passed to sublime.load_settings(). 215 | 216 | * none_erases (bool, optional) 217 | Iff ``True`` a setting's key will be erased when setting it to 218 | ``None``. This only has a meaning when the key you erase is 219 | defined in a parent Settings collection which would be 220 | retrieved in that case. 221 | 222 | Defines the following extra methods: 223 | 224 | save() 225 | Flushes in-memory changes to the disk 226 | 227 | See: sublime.save_settings(name) 228 | 229 | Adds these attributes to the list of unreferable attribute names for 230 | settings: 231 | 232 | ['_name', 'save'] 233 | 234 | Please compare with the list from sublime_lib.Settings or 235 | ``dir(FileSettings)``. 236 | """ 237 | _name = "" 238 | _settable_attributes = ('_s', '_name', '_none_erases') # allow only setting of these attributes 239 | 240 | def __init__(self, name, none_erases=False): 241 | settings = sublime.load_settings(name) 242 | if not settings: 243 | raise ValueError('Could not create settings from name "%s"' % name) 244 | self._name = name 245 | super(FileSettings, self).__init__(settings, none_erases) 246 | 247 | def save(self): 248 | sublime.save_settings(self._name) 249 | -------------------------------------------------------------------------------- /plugins/sublime/edit.py: -------------------------------------------------------------------------------- 1 | # edit.py, courtesy of @lunixbochs (https://github.com/lunixbochs) 2 | # and slightly modified 3 | # modified as well by @rageandqq 4 | """Abstraction for edit objects in ST3 5 | 6 | All methods on "edit" create an edit step. When leaving the `with` block, all 7 | the steps are executed one by one. 8 | 9 | Be careful: All other code in the with block is still executed! If a method on 10 | edit depends on something you do based on a previous method on edit, you 11 | should use the second method. However, using `edit.callback` or pass a 12 | function as an argument you can circumvent that if it's only small things. The 13 | function will be called with the parameters `view` and `edit` when processing 14 | the edit group. 15 | 16 | Usage 1: 17 | with Edit(view) as edit: 18 | edit.insert(0, "text") 19 | edit.replace(reg, "replacement") 20 | edit.erase(lambda v,e: sublime.Region(0, v.size())) 21 | # OR 22 | # edit.callback(lambda v,e: v.erase(e, sublime.Region(0, v.size()))) 23 | 24 | Usage 2: 25 | def do_ed(view, edit): 26 | edit.erase() 27 | view.insert(edit, 0, "text") 28 | view.sel().clear() 29 | view.sel().add(sublime.Region(0, 4)) 30 | edit.replace(reg, "replacement") 31 | 32 | Edit.call(do_ed) 33 | 34 | Available methods: 35 | Note: Any of these parameters can be a function which will be called with 36 | (optional) parameters `view` and `edit` when processing the edit group. 37 | Example callbacks: 38 | `lambda: 1` 39 | `lambda v: v.size()` 40 | `lambda v, e: v.erase(e, reg)` 41 | 42 | insert(point, string) 43 | view.insert(edit, point, string) 44 | 45 | append(point, string) 46 | view.insert(edit, view.size(), string) 47 | 48 | erase(region) 49 | view.erase(edit, region) 50 | 51 | replace(region, string) 52 | view.replace(edit, region, string) 53 | 54 | callback(func) 55 | func(view, edit) 56 | 57 | """ 58 | 59 | import inspect 60 | import sublime 61 | import sublime_plugin 62 | 63 | try: 64 | sublime.edit_storage 65 | except AttributeError: 66 | sublime.edit_storage = {} 67 | 68 | 69 | def run_callback(func, *args, **kwargs): 70 | spec = inspect.getfullargspec(func) 71 | 72 | args = args[:len(spec.args) or 0] 73 | if not spec.varargs: 74 | kwargs = {} 75 | 76 | return func(*args, **kwargs) 77 | 78 | 79 | class EditStep: 80 | def __init__(self, cmd, *args): 81 | self.cmd = cmd 82 | self.args = args 83 | 84 | def run(self, view, edit): 85 | if self.cmd == 'callback': 86 | return run_callback(self.args[0], view, edit) 87 | 88 | funcs = { 89 | 'insert': view.insert, 90 | 'erase': view.erase, 91 | 'replace': view.replace, 92 | } 93 | func = funcs.get(self.cmd) 94 | if func: 95 | args = self.resolve_args(view, edit) 96 | func(edit, *args) 97 | 98 | def resolve_args(self, view, edit): 99 | args = [] 100 | for arg in self.args: 101 | if callable(arg): 102 | arg = run_callback(arg, view, edit) 103 | args.append(arg) 104 | return args 105 | 106 | 107 | class Edit: 108 | def __init__(self, view, func=None): 109 | self.view = view 110 | self.steps = [] 111 | 112 | def __nonzero__(self): 113 | return bool(self.steps) 114 | 115 | __bool__ = __nonzero__ # Python 3 equivalent 116 | 117 | def step(self, cmd, *args): 118 | step = EditStep(cmd, *args) 119 | self.steps.append(step) 120 | 121 | def insert(self, point, string): 122 | self.step('insert', point, string) 123 | 124 | def append(self, string): 125 | # import spdb ; spdb.start() 126 | self.step('insert', lambda v: v.size(), string) 127 | 128 | def erase(self, region): 129 | self.step('erase', region) 130 | 131 | def replace(self, region, string): 132 | self.step('replace', region, string) 133 | 134 | @classmethod 135 | def call(cls, view, func): 136 | if not (func and callable(func)): 137 | return 138 | 139 | with cls(view) as edit: 140 | edit.callback(func) 141 | 142 | def callback(self, func): 143 | self.step('callback', func) 144 | 145 | def run(self, view, edit): 146 | for step in self.steps: 147 | step.run(view, edit) 148 | 149 | def __enter__(self): 150 | return self 151 | 152 | def __exit__(self, type, value, traceback): 153 | view = self.view 154 | key = str(hash(tuple(self.steps))) 155 | sublime.edit_storage[key] = self 156 | view.run_command('sl_apply_edit', {'key': key}) 157 | 158 | 159 | # Changed command name to not clash with other variations of this file 160 | class SlApplyEdit(sublime_plugin.TextCommand): 161 | def run(self, edit, key): 162 | sublime.edit_storage.pop(key).run(self.view, edit) 163 | 164 | 165 | # Make command known to sublime_command despite not being loaded by it 166 | sublime_plugin.text_command_classes.append(SlApplyEdit) 167 | 168 | # Make the command unloadable 169 | plugins = [SlApplyEdit] 170 | -------------------------------------------------------------------------------- /plugins/sublime/enum-dist/MANIFEST.in: -------------------------------------------------------------------------------- 1 | exclude enum/* 2 | include setup.py 3 | include README 4 | include enum/__init__.py 5 | include enum/test.py 6 | include enum/LICENSE 7 | include enum/README 8 | include enum/doc/enum.pdf 9 | include enum/doc/enum.rst 10 | -------------------------------------------------------------------------------- /plugins/sublime/enum-dist/PKG-INFO: -------------------------------------------------------------------------------- 1 | Metadata-Version: 1.1 2 | Name: enum34 3 | Version: 1.1.6 4 | Summary: Python 3.4 Enum backported to 3.3, 3.2, 3.1, 2.7, 2.6, 2.5, and 2.4 5 | Home-page: https://bitbucket.org/stoneleaf/enum34 6 | Author: Ethan Furman 7 | Author-email: ethan@stoneleaf.us 8 | License: BSD License 9 | Description: enum --- support for enumerations 10 | ======================================== 11 | 12 | An enumeration is a set of symbolic names (members) bound to unique, constant 13 | values. Within an enumeration, the members can be compared by identity, and 14 | the enumeration itself can be iterated over. 15 | 16 | from enum import Enum 17 | 18 | class Fruit(Enum): 19 | apple = 1 20 | banana = 2 21 | orange = 3 22 | 23 | list(Fruit) 24 | # [, , ] 25 | 26 | len(Fruit) 27 | # 3 28 | 29 | Fruit.banana 30 | # 31 | 32 | Fruit['banana'] 33 | # 34 | 35 | Fruit(2) 36 | # 37 | 38 | Fruit.banana is Fruit['banana'] is Fruit(2) 39 | # True 40 | 41 | Fruit.banana.name 42 | # 'banana' 43 | 44 | Fruit.banana.value 45 | # 2 46 | 47 | Repository and Issue Tracker at https://bitbucket.org/stoneleaf/enum34. 48 | 49 | Platform: UNKNOWN 50 | Classifier: Development Status :: 5 - Production/Stable 51 | Classifier: Intended Audience :: Developers 52 | Classifier: License :: OSI Approved :: BSD License 53 | Classifier: Programming Language :: Python 54 | Classifier: Topic :: Software Development 55 | Classifier: Programming Language :: Python :: 2.4 56 | Classifier: Programming Language :: Python :: 2.5 57 | Classifier: Programming Language :: Python :: 2.6 58 | Classifier: Programming Language :: Python :: 2.7 59 | Classifier: Programming Language :: Python :: 3.3 60 | Classifier: Programming Language :: Python :: 3.4 61 | Classifier: Programming Language :: Python :: 3.5 62 | Provides: enum 63 | -------------------------------------------------------------------------------- /plugins/sublime/enum-dist/README: -------------------------------------------------------------------------------- 1 | enum34 is the new Python stdlib enum module available in Python 3.4 2 | backported for previous versions of Python from 2.4 to 3.3. 3 | tested on 2.6, 2.7, and 3.3+ 4 | -------------------------------------------------------------------------------- /plugins/sublime/enum-dist/enum/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013, Ethan Furman. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions 6 | are met: 7 | 8 | Redistributions of source code must retain the above 9 | copyright notice, this list of conditions and the 10 | following disclaimer. 11 | 12 | Redistributions in binary form must reproduce the above 13 | copyright notice, this list of conditions and the following 14 | disclaimer in the documentation and/or other materials 15 | provided with the distribution. 16 | 17 | Neither the name Ethan Furman nor the names of any 18 | contributors may be used to endorse or promote products 19 | derived from this software without specific prior written 20 | permission. 21 | 22 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 | POSSIBILITY OF SUCH DAMAGE. 33 | -------------------------------------------------------------------------------- /plugins/sublime/enum-dist/enum/README: -------------------------------------------------------------------------------- 1 | enum34 is the new Python stdlib enum module available in Python 3.4 2 | backported for previous versions of Python from 2.4 to 3.3. 3 | tested on 2.6, 2.7, and 3.3+ 4 | -------------------------------------------------------------------------------- /plugins/sublime/enum-dist/enum/doc/enum.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/plugins/sublime/enum-dist/enum/doc/enum.pdf -------------------------------------------------------------------------------- /plugins/sublime/enum-dist/enum34.egg-info/PKG-INFO: -------------------------------------------------------------------------------- 1 | Metadata-Version: 1.1 2 | Name: enum34 3 | Version: 1.1.6 4 | Summary: Python 3.4 Enum backported to 3.3, 3.2, 3.1, 2.7, 2.6, 2.5, and 2.4 5 | Home-page: https://bitbucket.org/stoneleaf/enum34 6 | Author: Ethan Furman 7 | Author-email: ethan@stoneleaf.us 8 | License: BSD License 9 | Description: enum --- support for enumerations 10 | ======================================== 11 | 12 | An enumeration is a set of symbolic names (members) bound to unique, constant 13 | values. Within an enumeration, the members can be compared by identity, and 14 | the enumeration itself can be iterated over. 15 | 16 | from enum import Enum 17 | 18 | class Fruit(Enum): 19 | apple = 1 20 | banana = 2 21 | orange = 3 22 | 23 | list(Fruit) 24 | # [, , ] 25 | 26 | len(Fruit) 27 | # 3 28 | 29 | Fruit.banana 30 | # 31 | 32 | Fruit['banana'] 33 | # 34 | 35 | Fruit(2) 36 | # 37 | 38 | Fruit.banana is Fruit['banana'] is Fruit(2) 39 | # True 40 | 41 | Fruit.banana.name 42 | # 'banana' 43 | 44 | Fruit.banana.value 45 | # 2 46 | 47 | Repository and Issue Tracker at https://bitbucket.org/stoneleaf/enum34. 48 | 49 | Platform: UNKNOWN 50 | Classifier: Development Status :: 5 - Production/Stable 51 | Classifier: Intended Audience :: Developers 52 | Classifier: License :: OSI Approved :: BSD License 53 | Classifier: Programming Language :: Python 54 | Classifier: Topic :: Software Development 55 | Classifier: Programming Language :: Python :: 2.4 56 | Classifier: Programming Language :: Python :: 2.5 57 | Classifier: Programming Language :: Python :: 2.6 58 | Classifier: Programming Language :: Python :: 2.7 59 | Classifier: Programming Language :: Python :: 3.3 60 | Classifier: Programming Language :: Python :: 3.4 61 | Classifier: Programming Language :: Python :: 3.5 62 | Provides: enum 63 | -------------------------------------------------------------------------------- /plugins/sublime/enum-dist/enum34.egg-info/SOURCES.txt: -------------------------------------------------------------------------------- 1 | MANIFEST.in 2 | README 3 | setup.py 4 | enum/LICENSE 5 | enum/README 6 | enum/__init__.py 7 | enum/test.py 8 | enum/doc/enum.pdf 9 | enum/doc/enum.rst 10 | enum34.egg-info/PKG-INFO 11 | enum34.egg-info/SOURCES.txt 12 | enum34.egg-info/dependency_links.txt 13 | enum34.egg-info/top_level.txt -------------------------------------------------------------------------------- /plugins/sublime/enum-dist/enum34.egg-info/dependency_links.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /plugins/sublime/enum-dist/enum34.egg-info/top_level.txt: -------------------------------------------------------------------------------- 1 | enum 2 | -------------------------------------------------------------------------------- /plugins/sublime/enum-dist/setup.cfg: -------------------------------------------------------------------------------- 1 | [egg_info] 2 | tag_build = 3 | tag_date = 0 4 | tag_svn_revision = 0 5 | 6 | -------------------------------------------------------------------------------- /plugins/sublime/enum-dist/setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import setuptools 4 | from distutils.core import setup 5 | 6 | 7 | if sys.version_info[:2] < (2, 7): 8 | required = ['ordereddict'] 9 | else: 10 | required = [] 11 | 12 | long_desc = '''\ 13 | enum --- support for enumerations 14 | ======================================== 15 | 16 | An enumeration is a set of symbolic names (members) bound to unique, constant 17 | values. Within an enumeration, the members can be compared by identity, and 18 | the enumeration itself can be iterated over. 19 | 20 | from enum import Enum 21 | 22 | class Fruit(Enum): 23 | apple = 1 24 | banana = 2 25 | orange = 3 26 | 27 | list(Fruit) 28 | # [, , ] 29 | 30 | len(Fruit) 31 | # 3 32 | 33 | Fruit.banana 34 | # 35 | 36 | Fruit['banana'] 37 | # 38 | 39 | Fruit(2) 40 | # 41 | 42 | Fruit.banana is Fruit['banana'] is Fruit(2) 43 | # True 44 | 45 | Fruit.banana.name 46 | # 'banana' 47 | 48 | Fruit.banana.value 49 | # 2 50 | 51 | Repository and Issue Tracker at https://bitbucket.org/stoneleaf/enum34. 52 | ''' 53 | 54 | py2_only = () 55 | py3_only = () 56 | make = [ 57 | 'rst2pdf enum/doc/enum.rst --output=enum/doc/enum.pdf', 58 | ] 59 | 60 | 61 | data = dict( 62 | name='enum34', 63 | version='1.1.6', 64 | url='https://bitbucket.org/stoneleaf/enum34', 65 | packages=['enum'], 66 | package_data={ 67 | 'enum' : [ 68 | 'LICENSE', 69 | 'README', 70 | 'doc/enum.rst', 71 | 'doc/enum.pdf', 72 | 'test.py', 73 | ] 74 | }, 75 | license='BSD License', 76 | description='Python 3.4 Enum backported to 3.3, 3.2, 3.1, 2.7, 2.6, 2.5, and 2.4', 77 | long_description=long_desc, 78 | provides=['enum'], 79 | install_requires=required, 80 | author='Ethan Furman', 81 | author_email='ethan@stoneleaf.us', 82 | classifiers=[ 83 | 'Development Status :: 5 - Production/Stable', 84 | 'Intended Audience :: Developers', 85 | 'License :: OSI Approved :: BSD License', 86 | 'Programming Language :: Python', 87 | 'Topic :: Software Development', 88 | 'Programming Language :: Python :: 2.4', 89 | 'Programming Language :: Python :: 2.5', 90 | 'Programming Language :: Python :: 2.6', 91 | 'Programming Language :: Python :: 2.7', 92 | 'Programming Language :: Python :: 3.3', 93 | 'Programming Language :: Python :: 3.4', 94 | 'Programming Language :: Python :: 3.5', 95 | ], 96 | ) 97 | 98 | if __name__ == '__main__': 99 | setup(**data) 100 | -------------------------------------------------------------------------------- /plugins/sublime/sublime_dev_setup.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | SCRIPTPATH=$( cd $(dirname $0) ; pwd -P ) 4 | SUBLIME_PACKAGE_LOCATION="/Users/${USER}/Library/Application Support/Sublime Text 3/Packages" 5 | SUBLIME_PLUGIN_LOCATION="/Users/$USER/Library/Application Support/Sublime Text 3/Packages/tandem" 6 | 7 | UNINSTALL_OPTION="--uninstall" 8 | 9 | function install() { 10 | uninstall 11 | 12 | mkdir -p "${SUBLIME_PACKAGE_LOCATION}" 13 | mkdir "${SUBLIME_PLUGIN_LOCATION}" 14 | 15 | ln -s $SCRIPTPATH/../../agent "${SUBLIME_PLUGIN_LOCATION}" 16 | ln -s $SCRIPTPATH/../../crdt "${SUBLIME_PLUGIN_LOCATION}" 17 | ln -s $SCRIPTPATH/enum-dist "${SUBLIME_PLUGIN_LOCATION}" 18 | ln -s $SCRIPTPATH/*.py "${SUBLIME_PLUGIN_LOCATION}" 19 | ln -s $SCRIPTPATH/*.sublime-* "${SUBLIME_PLUGIN_LOCATION}" 20 | } 21 | 22 | function uninstall() { 23 | rm -rf "${SUBLIME_PLUGIN_LOCATION}" 24 | } 25 | 26 | if [ "$1" == "$UNINSTALL_OPTION" ] ; then 27 | uninstall 28 | else 29 | install 30 | fi 31 | -------------------------------------------------------------------------------- /plugins/sublime/tandem.sublime-commands: -------------------------------------------------------------------------------- 1 | [ 2 | { "caption": "Tandem: Start Session", "command": "tandem", "args": { "show_gui": "True" } }, 3 | { "caption": "Tandem: Join Existing Session", "command": "tandem_connect" }, 4 | { "caption": "Tandem: Show Session ID", "command": "tandem_session", "args": { "show_gui": "True" } }, 5 | { "caption": "Tandem: Leave Sesion", "command": "tandem_stop", "args": { "show_gui": "True" } } 6 | ] 7 | -------------------------------------------------------------------------------- /plugins/vim/README_nvim.md: -------------------------------------------------------------------------------- 1 | # Tandem 2 | 3 | Tandem is an add-on for your favorite text editor that enables peer-to-peer 4 | collaborative editing across different editors. 5 | 6 | This repository contains code for the Neovim plugin. For more details on 7 | Tandem, visit [our website](http://typeintandem.com), or our [mono-repository 8 | containing all other source code.](https://github.com/typeintandem/tandem) 9 | 10 | ## Installation 11 | To install, you will need to have a copy of the Neovim Python2 client. 12 | ``` 13 | pip install neovim 14 | ``` 15 | You must also have `python3` and `node.js` installed for the networking code to 16 | work. 17 | 18 | Neovim users have the option of installing in one of the following ways: 19 | - **[Recommended]** Using your favourite plugin manager (e.g. Vundle, vim-plug, 20 | etc.) Tandem should be compatible with most popular plugin managers 21 | - Installing Tandem directly. You’ll need download this repository to 22 | `~/.config/nvim/rplugin/python`. 23 | 24 | Whether you use a plugin manager or download directly, you will also need to 25 | complete an additional one-time step. 26 | Tandem uses Neovim remote plugin functionality for thread-safety, so you will 27 | need to: 28 | - launch `nvim` 29 | - run `:UpdateRemotePlugins` 30 | - quit `nvim` 31 | 32 | You are now ready to use Tandem! 33 | 34 | ## Usage 35 | Tandem users can choose either start a collaborative session or join an 36 | existing one. Starting a collaborative session will share the contents of your 37 | current buffer. Joining an existing session will open it’s contents in a new 38 | buffer. 39 | 40 | Please use one of the following commands: 41 | - `:Tandem` - creates a new tandem session and prints the session ID 42 | - `:Tandem ` - joins an existing tandem session with the specified 43 | session ID 44 | - `:TandemStop` - leaves the current session 45 | - `:TandemSession` - prints the current session ID 46 | 47 | It is recommended to leave the session before exiting neovim, but that process 48 | should be automated. 49 | 50 | ## Terms of Service 51 | By using Tandem, you agree that any modified versions of Tandem will not use 52 | the rendezvous server hosted by the owners. You must host and use your own copy 53 | of the rendezvous server. We want to provide a good user experience for Tandem, 54 | and it would be difficult to do that with modified clients as well. 55 | 56 | You can launch the rendezvous server by running `python3 ./rendezvous/main.py`. 57 | Change the address of the rendezvous server used by the agent in the 58 | configuration file to point to your server's host. This file is located at: 59 | `rplugin/python/tandem_lib/agent/tandem/agent/configuration.py` 60 | 61 | ## License 62 | Copyright (c) 2018 Team Lightly 63 | 64 | See [LICENSE.txt](LICENSE.txt) 65 | 66 | Licensed under the Apache License, Version 2.0 (the "License"); 67 | you may not use this file except in compliance with the License. 68 | You may obtain a copy of the License at: 69 | 70 | http://www.apache.org/licenses/LICENSE-2.0 71 | 72 | ## Authors 73 | Team Lightly 74 | [Geoffrey Yu](https://github.com/geoffxy), [Jamiboy 75 | Mohammad](https://github.com/jamiboym) and [Sameer 76 | Chitley](https://github.com/rageandqq) 77 | 78 | We are a team of senior Software Engineering students at the University of 79 | Waterloo. 80 | Tandem was created as our [Engineering Capstone Design 81 | Project](https://uwaterloo.ca/capstone-design). 82 | -------------------------------------------------------------------------------- /plugins/vim/README_vim.md: -------------------------------------------------------------------------------- 1 | # Tandem 2 | 3 | Tandem is an add-on for your favorite text editor that enables peer-to-peer 4 | collaborative editing across different editors. 5 | 6 | This repository contains code for the Vim plugin. For more details on Tandem, 7 | visit [our website](http://typeintandem.com), or our [mono-repository 8 | containing all other source code.](https://github.com/typeintandem/tandem) 9 | 10 | *Note: Vim is not officially supported due to its lack of thread-safety. 11 | Instead we recommend Tandem with Neovim, one of our officially supported 12 | plugins. 13 | We added functionality to this editor since it was a minimal amount of work to 14 | port the logic - please use at your own risk.* 15 | 16 | ## Installation 17 | To install, you must have a copy of vim compiled with python installed. 18 | You must also have `python3` and `node.js` installed. 19 | 20 | Vim users have the option of installing in one of the following ways: 21 | - **[Recommended]** Using your favourite plugin manager (e.g. Vundle, vim-plug, 22 | etc.) Tandem should be compatible with most popular plugin managers 23 | - Installing Tandem directly. You’ll need download this repository. In your 24 | `~/.vimrc`, make sure you source the `tandem_vim.vim` file: 25 | `source /path/to/tandem/plugin/tandem_vim.vim` 26 | 27 | ## Usage 28 | Tandem users can choose either start a collaborative session or join an 29 | existing one. Starting a collaborative session will share the contents of your 30 | current buffer. Joining an existing session will open it’s contents in a new 31 | buffer. 32 | 33 | Please use one of the following commands: 34 | - `:Tandem` - creates a new tandem session and prints the session ID 35 | - `:Tandem ` - joins an existing tandem session with the specified 36 | session ID 37 | - `:TandemStop` - leaves the current session 38 | - `:TandemSession` - prints the current session ID 39 | 40 | It is recommended to leave the session before exiting vim, but that process 41 | should be automated. 42 | 43 | ## Terms of Service 44 | By using Tandem, you agree that any modified versions of Tandem will not use 45 | the rendezvous server hosted by the owners. You must host and use your own copy 46 | of the rendezvous server. We want to provide a good user experience for Tandem, 47 | and it would be difficult to do that with modified clients as well. 48 | 49 | You can launch the rendezvous server by running `python3 ./rendezvous/main.py`. 50 | Change the address of the rendezvous server used by the agent in the 51 | configuration file to point to your server's host. This file is located at: 52 | `plugin/tandem_lib/agent/tandem/agent/configuration.py` 53 | 54 | ## License 55 | Copyright (c) 2018 Team Lightly 56 | 57 | See [LICENSE.txt](LICENSE.txt) 58 | 59 | Licensed under the Apache License, Version 2.0 (the "License"); 60 | you may not use this file except in compliance with the License. 61 | You may obtain a copy of the License at: 62 | 63 | http://www.apache.org/licenses/LICENSE-2.0 64 | ## Authors 65 | Team Lightly 66 | [Geoffrey Yu](https://github.com/geoffxy), [Jamiboy 67 | Mohammad](https://github.com/jamiboym) and [Sameer 68 | Chitley](https://github.com/rageandqq) 69 | 70 | We are a team of senior Software Engineering students at the University of 71 | Waterloo. 72 | Tandem was created as our [Engineering Capstone Design 73 | Project](https://uwaterloo.ca/capstone-design). 74 | -------------------------------------------------------------------------------- /plugins/vim/neovim_dev_setup.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | SCRIPTPATH=$( cd $(dirname $0) ; pwd -P ) 4 | NVIM_RPLUGIN_LOCATION="$HOME/.config/nvim/rplugin/python" 5 | UNINSTALL_OPTION="--uninstall" 6 | 7 | function install() { 8 | mkdir -p $NVIM_RPLUGIN_LOCATION 9 | ln -s $SCRIPTPATH/tandem_lib $NVIM_RPLUGIN_LOCATION/tandem_lib 10 | ln -s $SCRIPTPATH/tandem_neovim.py $NVIM_RPLUGIN_LOCATION/tandem_neovim.py 11 | 12 | echo "Symlinks created! Please open nvim and run :UpdateRemotePlugins to complete the install." 13 | } 14 | 15 | function uninstall() { 16 | rm $NVIM_RPLUGIN_LOCATION/tandem_lib 17 | rm $NVIM_RPLUGIN_LOCATION/tandem_neovim.py 18 | 19 | echo "Symlinks removed. Please open nvim and run :UpdateRemotePlugins to remove the plugin's bindings." 20 | } 21 | 22 | if [ "$1" == "$UNINSTALL_OPTION" ] ; then 23 | uninstall 24 | else 25 | install 26 | fi 27 | -------------------------------------------------------------------------------- /plugins/vim/tandem_lib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/plugins/vim/tandem_lib/__init__.py -------------------------------------------------------------------------------- /plugins/vim/tandem_lib/agent: -------------------------------------------------------------------------------- 1 | ../../../agent -------------------------------------------------------------------------------- /plugins/vim/tandem_neovim.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from threading import Event 4 | import neovim 5 | 6 | plugin_location = os.path.dirname(os.path.abspath(__file__)) 7 | if plugin_location not in sys.path: 8 | sys.path.insert(0, plugin_location) 9 | 10 | import tandem_lib.tandem_plugin as plugin 11 | import tandem_lib.agent.tandem.agent.protocol.messages.editor as m 12 | 13 | 14 | @neovim.plugin 15 | class TandemNeovimPlugin(object): 16 | def __init__(self, vim): 17 | self._vim = vim 18 | self._tandem = plugin.TandemPlugin( 19 | vim=vim, 20 | message_handler=self._handle_message, 21 | ) 22 | self._text_applied = Event() 23 | self._message = None 24 | 25 | @neovim.command("Tandem", nargs="*", sync=True) 26 | def start(self, args): 27 | session_id = args[0] if len(args) >= 1 else None 28 | self._tandem.start(session_id) 29 | self._session_id = session_id 30 | 31 | @neovim.command("TandemStop", nargs="*", sync=True) 32 | def stop(self, args): 33 | self._tandem.stop(invoked_from_autocmd=False) 34 | self._session_id = None 35 | 36 | @neovim.command("TandemSession", nargs="*", sync=False) 37 | def session(self, args): 38 | if not plugin.is_active: 39 | self._vim.async_call( 40 | lambda: self._vim.command('echom "No instance running."'), 41 | ) 42 | return 43 | self._vim.async_call( 44 | lambda: self._vim.command('echom "Session ID: {}"' 45 | .format(self._session_id)), 46 | ) 47 | 48 | @neovim.autocmd("VimLeave", sync=True) 49 | def on_vim_leave(self): 50 | self._tandem.stop(invoked_from_autocmd=True) 51 | 52 | @neovim.autocmd("TextChanged", sync=False) 53 | def on_text_changed(self): 54 | if not plugin.is_active: 55 | return 56 | self._tandem.check_buffer() 57 | 58 | @neovim.autocmd("TextChangedI", sync=False) 59 | def on_text_changed_i(self): 60 | if not plugin.is_active: 61 | return 62 | self._tandem.check_buffer() 63 | 64 | @neovim.function("TandemHandleWriteRequest", sync=True) 65 | def tandem_handle_write_request(self, args): 66 | try: 67 | self._tandem.handle_write_request(message=self._message) 68 | finally: 69 | self._vim.async_call(lambda: self._text_applied.set()) 70 | 71 | def _handle_message(self, message): 72 | if isinstance(message, m.WriteRequest): 73 | self._message = message 74 | self._text_applied.clear() 75 | self._vim.async_call( 76 | lambda: self._vim.funcs.TandemHandleWriteRequest(async=True), 77 | ) 78 | self._text_applied.wait() 79 | elif isinstance(message, m.SessionInfo): 80 | self._session_id = message.session_id 81 | self._vim.async_call( 82 | lambda: self._vim.command('echom "Session ID: {}"' 83 | .format(message.session_id)), 84 | ) 85 | -------------------------------------------------------------------------------- /plugins/vim/tandem_vim.vim: -------------------------------------------------------------------------------- 1 | if !has('python') 2 | " :echom is persistent messaging. See 3 | " http://learnvimscriptthehardway.stevelosh.com/chapters/01.html 4 | :echom 'ERROR: Please use a version of Vim with Python support' 5 | finish 6 | endif 7 | 8 | if !executable('python3') 9 | :echom 'ERROR: Global python3 install required.' 10 | finish 11 | endif 12 | 13 | " Bind the Tandem functions to globally available commands. 14 | " ================= 15 | " Start agent with `:Tandem` 16 | " Start agent and connect to network with `:Tandem ` 17 | com! -nargs=* Tandem py tandem_plugin.start() 18 | " ================ 19 | " Stop agent (and disconnect from network) with `:TandemStop` 20 | com! TandemStop py tandem_plugin.stop(False) 21 | 22 | " Show Session ID for active session 23 | com! TandemSession py tandem_plugin.show_session_id() 24 | 25 | " Get the absolute path to the folder this script resides in, respecting 26 | " symlinks 27 | let s:path = fnamemodify(resolve(expand(':p')), ':h') 28 | 29 | python << EOF 30 | 31 | import os 32 | import sys 33 | import vim 34 | 35 | # Add the script path to the python path 36 | local_path = vim.eval("s:path") 37 | if local_path not in sys.path: 38 | sys.path.insert(0, local_path) 39 | 40 | import tandem_lib.tandem_plugin as plugin 41 | import tandem_lib.agent.tandem.agent.protocol.messages.editor as m 42 | 43 | class TandemVimPlugin: 44 | def __init__(self): 45 | self._tandem = plugin.TandemPlugin( 46 | vim=vim, 47 | on_start=self._set_up_autocommands, 48 | message_handler=self._handle_message, 49 | ) 50 | self._message = None 51 | 52 | def _handle_message(self, message): 53 | self._message = message 54 | if isinstance(message, m.ApplyText): 55 | vim.command(":doautocmd User TandemApplyText") 56 | elif isinstance(message, m.WriteRequest): 57 | vim.command(":doautocmd User TandemWriteRequest") 58 | elif isinstance(message, m.SessionInfo): 59 | vim.command('echom "Session ID: {}"'.format(message.session_id)) 60 | self._session_id = message.session_id 61 | 62 | def _handle_apply_text(self): 63 | self._tandem.handle_apply_text(self._message) 64 | self._message = None 65 | 66 | def _handle_write_request(self): 67 | self._tandem.handle_write_request(self._message) 68 | self._message = None 69 | 70 | def _check_buffer(self): 71 | self._tandem.check_buffer() 72 | 73 | def _set_up_autocommands(self): 74 | vim.command(':autocmd!') 75 | vim.command('autocmd TextChanged py tandem_plugin._check_buffer()') 76 | vim.command('autocmd TextChangedI py tandem_plugin._check_buffer()') 77 | vim.command('autocmd VimLeave * py tandem_plugin.stop()') 78 | vim.command("autocmd User TandemApplyText py tandem_plugin._handle_apply_text()") 79 | vim.command("autocmd User TandemWriteRequest py tandem_plugin._handle_write_request()") 80 | 81 | def start(self, session_id=None): 82 | self._tandem.start(session_id) 83 | self._session_id = session_id 84 | 85 | def stop(self, invoked_from_autocmd=True): 86 | self._tandem.stop(invoked_from_autocmd) 87 | self._session_id = None 88 | 89 | def show_session_id(self): 90 | if not plugin.is_active: 91 | vim.command(':echom "No instance running."') 92 | return 93 | vim.command('echom "Session ID: {}"'.format(self._session_id)) 94 | 95 | 96 | tandem_plugin = TandemVimPlugin() 97 | 98 | EOF 99 | -------------------------------------------------------------------------------- /release_scripts/README.md: -------------------------------------------------------------------------------- 1 | # Release Scripts 2 | These scripts were created to simplify the release of Tandem plugins. 3 | 4 | 5 | ## Pre-requisites 6 | You should create a `.env` file locally with the credentials for the Tandem 7 | bot. Contact the Tandem maintainers for access. 8 | 9 | ## Usage 10 | The only script you'll need to run is the `tandem_release.sh` script. It takes 11 | in the type of plugin to build, and the path to (a clone of) the plugin 12 | repository as arguments. 13 | 14 | For example, to build the `vim` plugin, navigate: 15 | ``` 16 | ./tandem__release.sh vim /path/to/vim/repository/clone 17 | ``` 18 | (Note: You must be on `master` in this repository when committing) 19 | 20 | ## How it Works 21 | The `tandem_release.sh` script does the following: 22 | 23 | 1. "Setup": parses the arguments, determines variables required by parts of the script 24 | 1. "Prepare": runs a plugin specific `prepare.sh` -- copies the plugin and 25 | tandem agent code to the plugin repository 26 | 2. "Commit": commits the plugin code in the specified repository and pushes it 27 | to GitHub 28 | 3. "Release": runs a helper `release.py` that creates a GitHub release. Python 29 | is used to simplify interfacing with the GitHub API. 30 | 31 | The Tandem Bot credentials are used to commit and release the plugin. Your (the 32 | script runners) credentials will be used to push the committed code to GitHub. 33 | -------------------------------------------------------------------------------- /release_scripts/prepare_scripts/nvim.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Given a destination, this script builds the sublime plugin for distribution 4 | # at the destination. The destination supplied should be a local copy of the 5 | # plugin repository. 6 | 7 | if [[ "$1" == "" ]]; then 8 | echo "ERROR: Please supply a path to the plugin target destination." 9 | exit 1 10 | fi 11 | 12 | SCRIPT_PATH=$( cd $(dirname $0) ; pwd -P ) 13 | INSTALL_PATH="$1" 14 | 15 | # Make sure the path is up-to-date 16 | cd $INSTALL_PATH 17 | git pull origin master 18 | cd $SCRIPT_PATH 19 | 20 | # Clean existing items in plugin path 21 | rm -rf $INSTALL_PATH/*/ 22 | rm -f $INSTALL_PATH/* 23 | 24 | # Create plugin, lib, agent and crdt subdirectories 25 | mkdir $INSTALL_PATH/rplugin/ 26 | mkdir $INSTALL_PATH/rplugin/python/ 27 | mkdir $INSTALL_PATH/rplugin/python/tandem_lib/ 28 | mkdir $INSTALL_PATH/rplugin/python/tandem_lib/agent/ 29 | mkdir $INSTALL_PATH/rplugin/python/tandem_lib/crdt/ 30 | 31 | # Agent 32 | $( 33 | cd $SCRIPT_PATH/../../agent/; 34 | rm -f **/*.pyc 35 | ) 36 | cp -r $SCRIPT_PATH/../../agent/ $INSTALL_PATH/rplugin/python/tandem_lib/agent/ 37 | 38 | # CRDT 39 | $( 40 | cd $SCRIPT_PATH/../../crdt/; 41 | npm run clean; 42 | rm -rf node_modules; 43 | npm install; 44 | npm run build 45 | ) 46 | cp -r $SCRIPT_PATH/../../crdt/build/ $INSTALL_PATH/rplugin/python/tandem_lib/crdt/build/ 47 | 48 | # License 49 | cp $SCRIPT_PATH/../../LICENSE.txt $INSTALL_PATH 50 | 51 | # Neovim specific files 52 | cd $SCRIPT_PATH/../../plugins/vim/ 53 | cp tandem_lib/*.py $INSTALL_PATH/rplugin/python/tandem_lib/ 54 | cp tandem_neovim.py $INSTALL_PATH/rplugin/python 55 | cp README_nvim.md $INSTALL_PATH/README.md 56 | -------------------------------------------------------------------------------- /release_scripts/prepare_scripts/sublime.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Given a destination, this script builds the sublime plugin for distribution 4 | # at the destination. The destination supplied should be a local copy of the 5 | # plugin repository. 6 | 7 | if [[ "$1" == "" ]]; then 8 | echo "ERROR: Please supply a path to the plugin target destination." 9 | exit 1 10 | fi 11 | 12 | SCRIPT_PATH=$( cd $(dirname $0) ; pwd -P ) 13 | INSTALL_PATH="$1" 14 | 15 | # Make sure the path is up-to-date 16 | cd $INSTALL_PATH 17 | git pull origin master 18 | cd $SCRIPT_PATH 19 | 20 | # Clean existing items in plugin path 21 | rm -rf $INSTALL_PATH/*/ 22 | rm -f $INSTALL_PATH/* 23 | 24 | # Create agent and crdt subdirectories 25 | mkdir $INSTALL_PATH/agent/ 26 | mkdir $INSTALL_PATH/crdt/ 27 | 28 | # Agent 29 | $( 30 | cd $SCRIPT_PATH/../../agent/; 31 | rm -f **/*.pyc 32 | ) 33 | cp -r $SCRIPT_PATH/../../agent/ $INSTALL_PATH/agent/ 34 | 35 | # CRDT 36 | $( 37 | cd $SCRIPT_PATH/../../crdt/; 38 | npm run clean; 39 | rm -rf node_modules; 40 | npm install; 41 | npm run build 42 | ) 43 | cp -r $SCRIPT_PATH/../../crdt/build/ $INSTALL_PATH/crdt/build/ 44 | 45 | # License 46 | cp $SCRIPT_PATH/../../LICENSE.txt $INSTALL_PATH 47 | 48 | # Plugin specific files 49 | cd $SCRIPT_PATH/../../plugins/sublime/ 50 | cp -r enum-dist/ $INSTALL_PATH/enum-dist/ 51 | cp *.py $INSTALL_PATH 52 | cp *.sublime-* $INSTALL_PATH 53 | cp README.md $INSTALL_PATH 54 | 55 | # Required by Package Control 56 | touch $INSTALL_PATH/.no-sublime-package 57 | -------------------------------------------------------------------------------- /release_scripts/prepare_scripts/vim.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Given a destination, this script builds the sublime plugin for distribution 4 | # at the destination. The destination supplied should be a local copy of the 5 | # plugin repository. 6 | 7 | if [[ "$1" == "" ]]; then 8 | echo "ERROR: Please supply a path to the plugin target destination." 9 | exit 1 10 | fi 11 | 12 | SCRIPT_PATH=$( cd $(dirname $0) ; pwd -P ) 13 | INSTALL_PATH="$1" 14 | 15 | # Make sure the path is up-to-date 16 | cd $INSTALL_PATH 17 | git pull origin master 18 | cd $SCRIPT_PATH 19 | 20 | # Clean existing items in plugin path 21 | rm -rf $INSTALL_PATH/*/ 22 | rm -f $INSTALL_PATH/* 23 | 24 | # Create plugin, lib, agent and crdt subdirectories 25 | mkdir $INSTALL_PATH/plugin/ 26 | mkdir $INSTALL_PATH/plugin/tandem_lib/ 27 | mkdir $INSTALL_PATH/plugin/tandem_lib/agent/ 28 | mkdir $INSTALL_PATH/plugin/tandem_lib/crdt/ 29 | 30 | # Agent 31 | $( 32 | cd $SCRIPT_PATH/../../agent/; 33 | rm -f **/*.pyc 34 | ) 35 | cp -r $SCRIPT_PATH/../../agent/ $INSTALL_PATH/plugin/tandem_lib/agent/ 36 | 37 | # CRDT 38 | $( 39 | cd $SCRIPT_PATH/../../crdt/; 40 | npm run clean; 41 | rm -rf node_modules; 42 | npm install; 43 | npm run build 44 | ) 45 | cp -r $SCRIPT_PATH/../../crdt/build/ $INSTALL_PATH/plugin/tandem_lib/crdt/build/ 46 | 47 | # License 48 | cp $SCRIPT_PATH/../../LICENSE.txt $INSTALL_PATH 49 | 50 | # Plugin specific files 51 | cd $SCRIPT_PATH/../../plugins/vim/ 52 | cp tandem_lib/*.py $INSTALL_PATH/plugin/tandem_lib/ 53 | cp tandem_vim.vim $INSTALL_PATH/plugin/ 54 | cp README_vim.md $INSTALL_PATH/README.md 55 | -------------------------------------------------------------------------------- /release_scripts/release.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import semver 4 | from github import Github 5 | 6 | PLUGIN_TYPES = "[vim | nvim | sublime]" 7 | 8 | ORG_NAME = "typeintandem" 9 | 10 | VIM_REPO = "https://github.com/{}/vim".format(ORG_NAME) 11 | NVIM_REPO = "https://github.com/{}/nvim".format(ORG_NAME) 12 | SUBLIME_REPO = "https://github.com/{}/sublime".format(ORG_NAME) 13 | 14 | 15 | def error(msg): 16 | print("ERROR: {}.".format(msg)) 17 | exit(1) 18 | 19 | 20 | def main(): 21 | if len(sys.argv) < 2: 22 | error("Pass in plugin type as the first argument. " 23 | "Choose from: {}".format(PLUGIN_TYPES)) 24 | elif len(sys.argv) < 3: 25 | error("You must also pass in repository SHA as the argument") 26 | 27 | repo_type = sys.argv[1].lower() 28 | if repo_type == "sublime": 29 | repo_url = SUBLIME_REPO 30 | elif repo_type == "vim": 31 | repo_url = VIM_REPO 32 | elif repo_type == "nvim": 33 | repo_url = NVIM_REPO 34 | else: 35 | error("Please pass in one of {} as the plugin type" 36 | .format(PLUGIN_TYPES)) 37 | 38 | master_SHA = sys.argv[2] 39 | 40 | bot_username = os.environ.get("RELEASE_BOT_USERNAME") 41 | bot_password = os.environ.get("RELEASE_BOT_PASSWORD") 42 | 43 | g = Github(bot_username, bot_password) 44 | 45 | release_repo = None 46 | for repo in g.get_organization(ORG_NAME).get_repos(): 47 | if repo.html_url == repo_url: 48 | release_repo = repo 49 | break 50 | 51 | if release_repo is None: 52 | error("{} repo not found".format(repo_type)) 53 | 54 | tags = release_repo.get_tags() 55 | last_tag = None 56 | for t in tags: 57 | last_tag = t 58 | break 59 | 60 | if (last_tag is None): 61 | last_tag = '0.0.0' 62 | else: 63 | if last_tag.commit.sha == master_SHA: 64 | error("Cannot create release with same SHA") 65 | last_tag = last_tag.name 66 | 67 | tag = semver.bump_minor(last_tag) 68 | 69 | release_repo.create_git_tag_and_release( 70 | tag, 71 | "Release version {}".format(tag), 72 | "v{}".format(tag), 73 | "Release version {}".format(tag), 74 | master_SHA, 75 | "commit", 76 | ) 77 | 78 | print("Succesfully created release v{}".format(tag)) 79 | 80 | 81 | if __name__ == "__main__": 82 | main() 83 | -------------------------------------------------------------------------------- /release_scripts/tandem_release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # ======================================= 4 | # Setup 5 | # ======================================= 6 | set -e # terminate script if any command errors 7 | 8 | USAGE="Usage: ./tandem_release /path/to/repo" 9 | 10 | if ! [[ "$1" =~ ^(vim|nvim|sublime)$ ]]; then 11 | echo $USAGE 12 | echo "ERROR: Please supply a valid plugin type." 13 | exit 1 14 | fi 15 | 16 | if [[ "$2" == "" ]]; then 17 | echo $USAGE 18 | echo "ERROR: Please supply a path to the plugin target destination." 19 | exit 1 20 | fi 21 | 22 | RELEASE_SCRIPT_PATH=$( cd $(dirname $0) ; pwd -P ) 23 | 24 | cd $RELEASE_SCRIPT_PATH 25 | MASTER_HASH=$( git rev-parse master ) 26 | HASH=$( git rev-parse HEAD ) 27 | 28 | if [[ $MASTER_HASH != $HASH ]]; then 29 | echo "ERROR: You must be on master when releasing." 30 | exit 1 31 | fi 32 | 33 | # Clean all .pyc files and __pycache__ directories 34 | find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf 35 | 36 | 37 | PLUGIN_TYPE="$1" 38 | PLUGIN_TYPE_PATH="./$1/" 39 | TARGET_REPOSITORY_PATH="$2" 40 | 41 | # ======================================= 42 | # Prepare 43 | # ======================================= 44 | cd ./prepare_scripts 45 | ./$PLUGIN_TYPE.sh "../$TARGET_REPOSITORY_PATH" 46 | echo "Plugin files copied and prepared to commit." 47 | 48 | # ======================================= 49 | # Commit 50 | # ======================================= 51 | cd $RELEASE_SCRIPT_PATH 52 | MONOREPO_HASH=$( git rev-parse master ) 53 | 54 | cd $TARGET_REPOSITORY_PATH 55 | git add . 56 | 57 | # Prevent an error if HEAD doesn't exist 58 | NUM_COMMITS=$( git rev-list --count --all ) 59 | if [[ $NUM_COMMITS != "0" ]]; then 60 | CHANGED=$(git diff-index --name-only HEAD --) 61 | else 62 | CHANGED=1 63 | fi 64 | 65 | if [ -n "$CHANGED" ]; then 66 | git commit -m "Cut release from $MONOREPO_HASH" --author="Team Lightly " 67 | git push origin master # Repository should have the main remote set to "origin" 68 | echo "Release $MONOREPO_HASH authored and pushed." 69 | else 70 | echo "There were no changes to the repository and nothing was committed." 71 | # Does not exit - there might have been an error in releasing the last commit. 72 | # Allows the script to be used to retry the release. 73 | fi 74 | 75 | # ======================================= 76 | # Release 77 | # ======================================= 78 | echo -n "Are you sure you want to continue releasing (y/N)? " 79 | read answer 80 | if [[ $answer != "y" ]] && [[ $answer != "Y" ]] ;then 81 | exit 1 82 | fi 83 | 84 | cd $RELEASE_SCRIPT_PATH 85 | pip install pygithub & 86 | pip install semver & 87 | wait 88 | 89 | source .env 90 | 91 | cd $TARGET_REPOSITORY_PATH 92 | PLUGIN_REPO_HASH=$( git rev-parse master ) 93 | 94 | cd $RELEASE_SCRIPT_PATH 95 | python release.py $PLUGIN_TYPE $PLUGIN_REPO_HASH # python script used to simplify GitHub API usage 96 | -------------------------------------------------------------------------------- /rendezvous/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/rendezvous/__init__.py -------------------------------------------------------------------------------- /rendezvous/main.py: -------------------------------------------------------------------------------- 1 | import signal 2 | import logging 3 | import threading 4 | import argparse 5 | from tandem.rendezvous.executables.rendezvous import TandemRendezvous 6 | 7 | should_shutdown = threading.Event() 8 | 9 | 10 | def signal_handler(signal, frame): 11 | global should_shutdown 12 | should_shutdown.set() 13 | 14 | 15 | def set_up_logging(log_location): 16 | logging.basicConfig( 17 | level=logging.DEBUG, 18 | format="%(asctime)s %(levelname)-8s %(message)s", 19 | datefmt="%Y-%m-%d %H:%M", 20 | filename=log_location, 21 | filemode="w", 22 | ) 23 | 24 | 25 | def main(): 26 | signal.signal(signal.SIGINT, signal_handler) 27 | signal.signal(signal.SIGTERM, signal_handler) 28 | 29 | parser = argparse.ArgumentParser( 30 | description="Starts the Tandem rendezvous server." 31 | ) 32 | parser.add_argument( 33 | "--host", 34 | default="", 35 | help="The host address to bind to.", 36 | ) 37 | parser.add_argument( 38 | "--port", 39 | default=60000, 40 | type=int, 41 | help="The port to listen on.", 42 | ) 43 | parser.add_argument( 44 | "--log-file", 45 | default="/tmp/tandem-rendezvous.log", 46 | help="The location of the log file.", 47 | ) 48 | args = parser.parse_args() 49 | 50 | set_up_logging(args.log_file) 51 | 52 | # Run the rendezvous server until asked to terminate 53 | with TandemRendezvous(args.host, args.port): 54 | should_shutdown.wait() 55 | 56 | 57 | if __name__ == "__main__": 58 | main() 59 | -------------------------------------------------------------------------------- /rendezvous/prototype/client.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import socket 3 | import json 4 | import time 5 | 6 | 7 | def get_args(): 8 | parser = argparse.ArgumentParser() 9 | parser.add_argument("--target_host", default="127.0.0.1") 10 | parser.add_argument("--target_port", default=60000, type=int) 11 | parser.add_argument("--self_port", default=60001, type=int) 12 | return parser.parse_args() 13 | 14 | 15 | def recv_data(sock): 16 | print("Waiting to receive data") 17 | data, address = sock.recvfrom(4096) 18 | print("Received data: {} from address: {}".format(data, address)) 19 | return json.loads(data), address 20 | 21 | 22 | def send_data(sock, data, address): 23 | print("Sending data: {} to address: {}".format(data, address)) 24 | sock.sendto(json.dumps(data), address) 25 | 26 | 27 | def connect_to(sock, data): 28 | print("Sending ping to addresses: {}".format(data)) 29 | send_data(sock, create_ping(data[1]), tuple(data[0])) 30 | send_data(sock, create_ping(data[1]), tuple(data[1])) 31 | 32 | 33 | def create_ping(address): 34 | return { 35 | 'type': 'ping', 36 | 'address': address, 37 | } 38 | 39 | 40 | def create_pingback(ping): 41 | return { 42 | 'type': 'pingback', 43 | 'address': ping['address'], 44 | } 45 | 46 | 47 | def main(): 48 | args = get_args() 49 | self_address = (socket.gethostbyname(socket.gethostname()), args.self_port) 50 | server_address = (args.target_host, args.target_port) 51 | 52 | print("Listening on {}".format(self_address)) 53 | sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 54 | sock.bind(self_address) 55 | 56 | print("Connecting to rendezvous server") 57 | send_data(sock, self_address, server_address) 58 | 59 | while(True): 60 | data, address = recv_data(sock) 61 | 62 | if (type(data) is list and type(data[0]) is list): 63 | connect_to(sock, data) 64 | else: 65 | if data['type'] == 'ping': 66 | time.sleep(1) 67 | send_data(sock, create_pingback(data), address) 68 | 69 | 70 | main() 71 | -------------------------------------------------------------------------------- /rendezvous/prototype/server.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import socket 3 | import json 4 | import time 5 | 6 | 7 | def get_args(): 8 | parser = argparse.ArgumentParser() 9 | parser.add_argument("--self_port", default=60000, type=int) 10 | return parser.parse_args() 11 | 12 | 13 | def recv_data(sock): 14 | print("Waiting to receive data") 15 | data, address = sock.recvfrom(4096) 16 | print("Received data: {} from address: {}".format(data, address)) 17 | return json.loads(data), address 18 | 19 | 20 | def send_data(sock, data, address): 21 | print("Sending data: {} to address: {}".format(data, address)) 22 | sock.sendto(json.dumps(data), address) 23 | 24 | 25 | def main(): 26 | args = get_args() 27 | self_address = (socket.gethostbyname(socket.gethostname()), args.self_port) 28 | connected_clients = [] 29 | 30 | print("Listening on {}".format(self_address)) 31 | sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 32 | sock.bind(self_address) 33 | 34 | while(True): 35 | new_data, new_address = recv_data(sock) 36 | 37 | # Send new client information about the connected_clients 38 | for connected_data, connected_address in connected_clients: 39 | send_data(sock, (connected_data, connected_address), new_address) 40 | time.sleep(3) 41 | 42 | time.sleep(3) 43 | 44 | # Send connected_clients information about the new client 45 | for connected_data, connected_address in connected_clients: 46 | send_data(sock, (new_data, new_address), connected_address) 47 | 48 | connected_clients.append((new_data, new_address)) 49 | 50 | 51 | main() 52 | -------------------------------------------------------------------------------- /rendezvous/tandem-rendezvous.conf: -------------------------------------------------------------------------------- 1 | description "Tandem Rendezvous Server" 2 | 3 | start on filesystem or runlevel [2345] 4 | stop on shutdown 5 | 6 | # Respawn on crash but give up after 10 retries within 5 seconds 7 | respawn 8 | respawn limit 10 5 9 | 10 | exec python3 /home/ec2-user/tandem/rendezvous/main.py \ 11 | --port 60000 \ 12 | --log-file /var/log/tandem-rendezvous-$(date +%s).log 13 | -------------------------------------------------------------------------------- /rendezvous/tandem/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/rendezvous/tandem/__init__.py -------------------------------------------------------------------------------- /rendezvous/tandem/rendezvous/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/rendezvous/tandem/rendezvous/__init__.py -------------------------------------------------------------------------------- /rendezvous/tandem/rendezvous/executables/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/rendezvous/tandem/rendezvous/executables/__init__.py -------------------------------------------------------------------------------- /rendezvous/tandem/rendezvous/executables/rendezvous.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from tandem.shared.io.udp_gateway import UDPGateway 3 | from tandem.shared.io.proxies.fragment import FragmentProxy 4 | from tandem.shared.io.proxies.list_parameters import ListParametersProxy 5 | from tandem.shared.io.proxies.unicode import UnicodeProxy 6 | from tandem.rendezvous.io.proxies.relay import RendezvousRelayProxy 7 | from tandem.shared.io.proxies.reliability import ReliabilityProxy 8 | from tandem.rendezvous.protocol.handlers.agent import ( 9 | AgentRendezvousProtocolHandler 10 | ) 11 | from tandem.shared.utils.time_scheduler import TimeScheduler 12 | from concurrent.futures import ThreadPoolExecutor 13 | 14 | 15 | class TandemRendezvous(object): 16 | def __init__(self, host, port): 17 | self._main_executor = ThreadPoolExecutor(max_workers=1) 18 | self._time_scheduler = TimeScheduler(self._main_executor) 19 | self._udp_gateway = UDPGateway( 20 | host, 21 | port, 22 | self._on_receive_message, 23 | [ 24 | ListParametersProxy(), 25 | UnicodeProxy(), 26 | FragmentProxy(), 27 | RendezvousRelayProxy(), 28 | ReliabilityProxy(self._time_scheduler), 29 | ], 30 | ) 31 | self._rendezvous_protocol = AgentRendezvousProtocolHandler( 32 | self._udp_gateway, 33 | ) 34 | 35 | def __enter__(self): 36 | self.start() 37 | return self 38 | 39 | def __exit__(self, exc_type, exc_value, traceback): 40 | self.stop() 41 | 42 | def start(self): 43 | self._time_scheduler.start() 44 | self._udp_gateway.start() 45 | logging.info("Tandem Rendezvous has started.") 46 | 47 | def stop(self): 48 | self._udp_gateway.stop() 49 | self._time_scheduler.stop() 50 | self._main_executor.shutdown() 51 | logging.info("Tandem Rendezvous has shut down.") 52 | 53 | def _on_receive_message(self, retrieve_data): 54 | self._main_executor.submit( 55 | self._rendezvous_protocol.handle_raw_data, 56 | retrieve_data, 57 | ) 58 | -------------------------------------------------------------------------------- /rendezvous/tandem/rendezvous/io/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/rendezvous/tandem/rendezvous/io/__init__.py -------------------------------------------------------------------------------- /rendezvous/tandem/rendezvous/io/proxies/relay.py: -------------------------------------------------------------------------------- 1 | from tandem.shared.io.proxies.base import ProxyBase 2 | from tandem.shared.utils.relay import RelayUtils 3 | from tandem.rendezvous.stores.session import SessionStore 4 | 5 | 6 | class RendezvousRelayProxy(ProxyBase): 7 | def on_retrieve_io_data(self, params): 8 | args, kwargs = params 9 | if args is None or args is (None, None): 10 | return params 11 | 12 | raw_data, in_address = args 13 | 14 | if RelayUtils.is_relay(raw_data): 15 | payload, out_address = RelayUtils.deserialize(raw_data) 16 | 17 | # Check that the peers belong in the same session 18 | session_store = SessionStore.get_instance() 19 | in_session = session_store.get_session_from_address(in_address) 20 | out_session = session_store.get_session_from_address(out_address) 21 | if in_session != out_session: 22 | return (None, None) 23 | 24 | new_data = RelayUtils.serialize(payload, in_address) 25 | 26 | # HACK: Write the data directly 27 | self._interface.write_io_data( 28 | [self._interface.data_class(new_data, out_address)], 29 | reliability=True, 30 | ) 31 | 32 | return (None, None) 33 | else: 34 | return params 35 | -------------------------------------------------------------------------------- /rendezvous/tandem/rendezvous/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/rendezvous/tandem/rendezvous/models/__init__.py -------------------------------------------------------------------------------- /rendezvous/tandem/rendezvous/models/connection.py: -------------------------------------------------------------------------------- 1 | from tandem.shared.models.base import ModelBase 2 | 3 | 4 | class Connection(ModelBase): 5 | def __init__(self, peer): 6 | self._peer = peer 7 | 8 | def __eq__(self, other): 9 | return self._peer == other._peer 10 | 11 | def get_id(self): 12 | return self._peer.get_id() 13 | 14 | def get_peer(self): 15 | return self._peer 16 | -------------------------------------------------------------------------------- /rendezvous/tandem/rendezvous/models/session.py: -------------------------------------------------------------------------------- 1 | from tandem.shared.models.base import ModelBase 2 | 3 | 4 | class Session(ModelBase): 5 | def __init__(self, session_id): 6 | self._connections = {} 7 | self._session_id = session_id 8 | 9 | def add_connection(self, connection): 10 | self._connections[connection.get_id()] = connection 11 | 12 | def remove_connection(self, connection): 13 | del self._connections[connection.get_id()] 14 | 15 | def get_connection(self, id): 16 | self._connections.get(id, None) 17 | 18 | def get_connections(self): 19 | return [connection for _, connection in self._connections.items()] 20 | -------------------------------------------------------------------------------- /rendezvous/tandem/rendezvous/protocol/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/rendezvous/tandem/rendezvous/protocol/__init__.py -------------------------------------------------------------------------------- /rendezvous/tandem/rendezvous/protocol/handlers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/rendezvous/tandem/rendezvous/protocol/handlers/__init__.py -------------------------------------------------------------------------------- /rendezvous/tandem/rendezvous/protocol/handlers/agent.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | import logging 3 | from tandem.rendezvous.models.connection import Connection 4 | from tandem.rendezvous.stores.session import SessionStore 5 | from tandem.shared.models.peer import Peer 6 | from tandem.shared.protocol.handlers.addressed import AddressedHandler 7 | from tandem.shared.protocol.messages.rendezvous import ( 8 | RendezvousProtocolMessageType, 9 | RendezvousProtocolUtils, 10 | SetupParameters, 11 | Error, 12 | ) 13 | from tandem.shared.utils.static_value import static_value as staticvalue 14 | 15 | 16 | def parse_uuid(candidate): 17 | try: 18 | return uuid.UUID(candidate) 19 | except ValueError: 20 | return None 21 | 22 | 23 | class AgentRendezvousProtocolHandler(AddressedHandler): 24 | @staticvalue 25 | def _protocol_message_utils(self): 26 | return RendezvousProtocolUtils 27 | 28 | @staticvalue 29 | def _protocol_message_handlers(self): 30 | return { 31 | RendezvousProtocolMessageType.ConnectRequest.value: 32 | self._handle_connect_request, 33 | } 34 | 35 | def __init__(self, gateway): 36 | self._gateway = gateway 37 | 38 | def _handle_connect_request(self, message, sender_address): 39 | # Validate request identifiers 40 | connection_id = parse_uuid(message.my_id) 41 | session_id = parse_uuid(message.session_id) 42 | if connection_id is None or session_id is None: 43 | logging.info( 44 | "Rejecting ConnectRequest from {}:{} due to malformed" 45 | " connection and/or session id." 46 | .format(*sender_address), 47 | ) 48 | self._send_error_message(sender_address, "Invalid ids.") 49 | return 50 | 51 | # Fetch or create the session 52 | session = SessionStore.get_instance().get_or_create_session(session_id) 53 | 54 | # Make sure the agent requesting to join is new or has the same 55 | # "identity" as an agent already in the session. 56 | initiator = Connection(Peer( 57 | id=connection_id, 58 | public_address=sender_address, 59 | private_address=message.private_address, 60 | )) 61 | existing_connection = session.get_connection(connection_id) 62 | if existing_connection is None: 63 | session.add_connection(initiator) 64 | elif not(initiator == existing_connection): 65 | # Reject the connection request for security reasons since the 66 | # client gets to choose their ID. The first agent to join a 67 | # session "claims" the ID. This is not foolproof, but it makes 68 | # it more difficult for someone to join an existing session as 69 | # someone else. 70 | logging.info( 71 | "Rejecting ConnectRequest from {}:{} due to existing" 72 | " connection with the same id." 73 | .format(*sender_address), 74 | ) 75 | self._send_error_message(sender_address, "Invalid session.") 76 | return 77 | 78 | logging.info( 79 | "Connection {} is joining session {} requested by {}:{}".format( 80 | str(connection_id), 81 | str(session_id), 82 | *sender_address, 83 | ), 84 | ) 85 | 86 | for member_connection in session.get_connections(): 87 | if not(member_connection == initiator): 88 | # Send initiator's details to the session member 89 | self._send_setup_parameters_message( 90 | session_id=session_id, 91 | recipient=member_connection, 92 | should_connect_to=initiator, 93 | initiate=False, 94 | ) 95 | 96 | # Send the session member's details to the initiator 97 | self._send_setup_parameters_message( 98 | session_id=session_id, 99 | recipient=initiator, 100 | should_connect_to=member_connection, 101 | initiate=True, 102 | ) 103 | 104 | def _send_setup_parameters_message( 105 | self, 106 | session_id, 107 | recipient, 108 | should_connect_to, 109 | initiate, 110 | ): 111 | io_data = self._gateway.generate_io_data( 112 | self._protocol_message_utils().serialize(SetupParameters( 113 | session_id=str(session_id), 114 | peer_id=str(should_connect_to.get_peer().get_id()), 115 | initiate=initiate, 116 | public=should_connect_to.get_peer().get_public_address(), 117 | private=should_connect_to.get_peer().get_private_address(), 118 | )), 119 | recipient.get_peer().get_public_address(), 120 | ) 121 | self._gateway.write_io_data(io_data) 122 | 123 | def _send_error_message(self, address, message): 124 | io_data = self._gateway.generate_io_data( 125 | self._protocol_message_utils().serialize(Error( 126 | message=message, 127 | )), 128 | address, 129 | ) 130 | self._gateway.write_io_data(io_data) 131 | -------------------------------------------------------------------------------- /rendezvous/tandem/rendezvous/stores/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeintandem/tandem/81e76f675634f1b42c8c3070c73443f3f68f8624/rendezvous/tandem/rendezvous/stores/__init__.py -------------------------------------------------------------------------------- /rendezvous/tandem/rendezvous/stores/session.py: -------------------------------------------------------------------------------- 1 | from tandem.shared.stores.base import StoreBase 2 | from tandem.rendezvous.models.session import Session 3 | 4 | 5 | class SessionStore(StoreBase): 6 | def __init__(self): 7 | self._sessions = {} 8 | 9 | def get_or_create_session(self, session_id): 10 | session = self._sessions.get(session_id, None) 11 | if session is None: 12 | session = Session(session_id) 13 | self._sessions[session_id] = session 14 | return session 15 | 16 | def get_session_from_address(self, address): 17 | for _, session in self._sessions.items(): 18 | addresses = [ 19 | connection.get_peer().get_public_address() 20 | for connection in session.get_connections() 21 | ] 22 | if address in addresses: 23 | return session 24 | return None 25 | -------------------------------------------------------------------------------- /rendezvous/tandem/shared: -------------------------------------------------------------------------------- 1 | ../../lib-python/ -------------------------------------------------------------------------------- /rendezvous/test_client.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | import socket 3 | from tandem.shared.protocol.messages.rendezvous import ( 4 | RendezvousProtocolUtils, 5 | ConnectRequest, 6 | ) 7 | 8 | 9 | def recv_data(sock): 10 | print("Waiting to receive data") 11 | data, address = sock.recvfrom(4096) 12 | print("Received data: {} from address: {}".format(data, address)) 13 | return data, address 14 | 15 | 16 | def send_data(sock, data, address): 17 | print("Sending data: {} to address: {}".format(data, address)) 18 | sock.sendto(data.encode("utf-8"), address) 19 | 20 | 21 | def test_create_and_join_existing_session(sock, server_address, self_address): 22 | print("===== Test Creation and Join =====") 23 | peer1_id = uuid.uuid4() 24 | session_id = uuid.uuid4() 25 | 26 | send_data(sock, RendezvousProtocolUtils.serialize(ConnectRequest( 27 | session_id=str(session_id), 28 | my_id=str(peer1_id), 29 | private_address=self_address, 30 | )), server_address) 31 | 32 | peer2_address = (socket.gethostbyname("localhost"), 60002) 33 | peer2_id = uuid.uuid4() 34 | sock2 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 35 | sock2.bind(peer2_address) 36 | 37 | send_data(sock2, RendezvousProtocolUtils.serialize(ConnectRequest( 38 | my_id=str(peer2_id), 39 | session_id=str(session_id), 40 | private_address=peer2_address, 41 | )), server_address) 42 | 43 | print("--- Peer 1 Received:") 44 | recv_data(sock) 45 | 46 | print("--- Peer 2 Received:") 47 | recv_data(sock2) 48 | sock2.close() 49 | 50 | 51 | def test_invalid_id(sock, server_address, self_address): 52 | print("===== Test Invalid ID =====") 53 | send_data(sock, RendezvousProtocolUtils.serialize(ConnectRequest( 54 | session_id="123", 55 | my_id="123", 56 | private_address=self_address, 57 | )), server_address) 58 | recv_data(sock) 59 | 60 | 61 | def main(): 62 | self_address = (socket.gethostbyname("localhost"), 60001) 63 | server_address = (socket.gethostbyname("localhost"), 60000) 64 | 65 | print("Listening on {}".format(self_address)) 66 | sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 67 | sock.bind(self_address) 68 | 69 | print("Connecting to rendezvous server") 70 | test_create_and_join_existing_session(sock, server_address, self_address) 71 | test_invalid_id(sock, server_address, self_address) 72 | 73 | 74 | if __name__ == "__main__": 75 | main() 76 | --------------------------------------------------------------------------------