├── tests ├── __init__.py └── test_asgi_ipc.py ├── setup.cfg ├── .gitignore ├── tox.ini ├── asgi_ipc ├── __init__.py ├── core.py └── store.py ├── .travis.yml ├── Makefile ├── CHANGELOG.txt ├── setup.py ├── LICENSE └── README.rst /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal=1 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | *.pyc 3 | dist/ 4 | build/ 5 | /.tox 6 | __pycache__ 7 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27,py34,py35,py36 3 | 4 | [testenv] 5 | commands = python -m unittest discover 6 | -------------------------------------------------------------------------------- /asgi_ipc/__init__.py: -------------------------------------------------------------------------------- 1 | import pkg_resources 2 | __version__ = pkg_resources.require('asgi_ipc')[0].version 3 | 4 | from .core import IPCChannelLayer # noqa 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | dist: trusty 3 | language: python 4 | python: 5 | - "2.7" 6 | - "3.4" 7 | - "3.5" 8 | - "3.6" 9 | install: 10 | - pip install -e . 11 | script: 12 | - python -m unittest discover 13 | -------------------------------------------------------------------------------- /tests/test_asgi_ipc.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from asgi_ipc import IPCChannelLayer 4 | from asgiref.conformance import ConformanceTestCase 5 | 6 | 7 | # Default conformance tests 8 | class IPCLayerTests(ConformanceTestCase): 9 | 10 | channel_layer = IPCChannelLayer(expiry=1, group_expiry=2, capacity=5) 11 | expiry_delay = 1.1 12 | capacity_limit = 5 13 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: release 2 | 3 | all: 4 | 5 | release: 6 | ifndef version 7 | $(error Please supply a version) 8 | endif 9 | @echo Releasing version $(version) 10 | ifeq (,$(findstring $(version),$(shell git log --oneline -1))) 11 | $(error Last commit does not match version) 12 | endif 13 | git tag $(version) 14 | git push 15 | git push --tags 16 | python setup.py sdist bdist_wheel upload 17 | -------------------------------------------------------------------------------- /CHANGELOG.txt: -------------------------------------------------------------------------------- 1 | 1.4.2 (2017-10-26) 2 | ------------------ 3 | 4 | * Fix KeyError for empty groups 5 | 6 | 7 | 1.4.1 (2017-08-11) 8 | ------------------ 9 | 10 | * Fix empty shared memory area bug 11 | 12 | 13 | 1.4.0 (2017-05-24) 14 | ------------------ 15 | 16 | * The message storage backend has been overhauled to lock and serialize less, 17 | improving performance considerably and removing all double-receive bugs. 18 | Throughput on even a low-powered laptop should be in the thousands of messages 19 | per second. 20 | 21 | 1.3.1 (2017-04-02) 22 | ------------------ 23 | 24 | * Error with sending to multi-process channels with the same message fixed 25 | 26 | 1.3.0 (2017-04-01) 27 | ------------------ 28 | 29 | * Updated to new process-specific channel style 30 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from setuptools import setup 3 | 4 | __version__ = '1.4.2' 5 | 6 | # We use the README as the long_description 7 | readme_path = os.path.join(os.path.dirname(__file__), "README.rst") 8 | 9 | 10 | setup( 11 | name='asgi_ipc', 12 | version=__version__, 13 | url='http://github.com/django/asgi_ipc/', 14 | author='Django Software Foundation', 15 | author_email='foundation@djangoproject.com', 16 | description='Posix IPC-backed ASGI channel layer implementation', 17 | long_description=open(readme_path).read(), 18 | license='BSD', 19 | zip_safe=False, 20 | packages=["asgi_ipc"], 21 | include_package_data=True, 22 | install_requires=[ 23 | 'six', 24 | 'posix_ipc>=1.0.0', 25 | 'msgpack-python', 26 | 'asgiref~=1.1.2', 27 | ] 28 | ) 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) Django Software Foundation and individual contributors. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of Django nor the names of its contributors may be used 15 | to endorse or promote products derived from this software without 16 | specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 25 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | asgi_ipc 2 | ======== 3 | 4 | .. image:: https://api.travis-ci.org/django/asgi_ipc.svg 5 | :target: https://travis-ci.org/django/asgi_ipc 6 | 7 | .. image:: https://img.shields.io/pypi/v/asgi_ipc.svg 8 | :target: https://pypi.python.org/pypi/asgi_ipc 9 | 10 | **NOTE: This is for Channels 1 only. It will not work on Channels 2 and there's no plans to port it.** 11 | 12 | An ASGI channel layer that uses POSIX shared memory IPC as its backing store 13 | (only works between processes on the same machine). 14 | 15 | IPC is still a bit of a wild west of UNIX compatibility, so if you find weird 16 | errors, please file an issue with details and exact system specifications. In 17 | particular, this module is tested and works well on Linux kernels and the Windows 18 | Subsystem for Linux; it also works on Mac OS but will not be able to detect 19 | deadlock situations due to limitations in the kernel. 20 | 21 | 22 | Usage 23 | ----- 24 | 25 | You'll need to instantiate the channel layer with a path prefix to create 26 | IPC objects underneath; any channel layers with the same prefix will talk to 27 | each other as long as they're on the same machine. 28 | 29 | Example: 30 | 31 | .. code-block:: python 32 | 33 | import asgi_ipc as asgi 34 | 35 | channel_layer = asgi.IPCChannelLayer( 36 | prefix="aeracode", 37 | message_memory=200 * 1024 * 1024, 38 | ) 39 | 40 | channel_layer.send("my_channel", {"text": "Hello ASGI"}) 41 | print(channel_layer.receive(["my_channel", ])) 42 | 43 | ``prefix`` 44 | ~~~~~~~~~~ 45 | 46 | Prefix to use for IPC objects under the root namespace. Defaults to ``asgi``. 47 | IPC layers on the same machine with the same prefix will talk to each other. 48 | 49 | ``message_memory`` 50 | ~~~~~~~~~~~~~~~~~~ 51 | 52 | The amount of shared memory to allocate to the channel storage, in bytes. 53 | Defaults to 100MB. All of your in-flight messages must fit into this, 54 | otherwise you'll get ``ChannelFull`` errors if the memory space is full up. 55 | 56 | ASGI messages can be a maximum of one megabyte, and are usually much smaller. 57 | The IPC routing metadata on top of each message is approximately 50 bytes. 58 | 59 | ``group_memory`` 60 | ~~~~~~~~~~~~~~~~ 61 | 62 | The amount of shared memory to allocate to the group storage, in bytes. 63 | Defaults to 20MB. All of your group membership data must fit into this space, 64 | otherwise your group memberships may fail to persist. 65 | 66 | You can fit approximately 4000 group-channel membership associations into one 67 | megabyte of memory. 68 | 69 | ``expiry`` 70 | ~~~~~~~~~~ 71 | 72 | Message expiry in seconds. Defaults to ``60``. You generally shouldn't need 73 | to change this, but you may want to turn it down if you have peaky traffic you 74 | wish to drop, or up if you have peaky traffic you want to backlog until you 75 | get to it. 76 | 77 | ``group_expiry`` 78 | ~~~~~~~~~~~~~~~~ 79 | 80 | Group expiry in seconds. Defaults to ``86400``. Interface servers will drop 81 | connections after this amount of time; it's recommended you reduce it for a 82 | healthier system that encourages disconnections. 83 | 84 | ``capacity`` 85 | ~~~~~~~~~~~~ 86 | 87 | Default channel capacity. Defaults to ``100``. Once a channel is at capacity, 88 | it will refuse more messages. How this affects different parts of the system 89 | varies; a HTTP server will refuse connections, for example, while Django 90 | sending a response will just wait until there's space. 91 | 92 | ``channel_capacity`` 93 | ~~~~~~~~~~~~~~~~~~~~ 94 | 95 | Per-channel capacity configuration. This lets you tweak the channel capacity 96 | based on the channel name, and supports both globbing and regular expressions. 97 | 98 | It should be a dict mapping channel name pattern to desired capacity; if the 99 | dict key is a string, it's interpreted as a glob, while if it's a compiled 100 | ``re`` object, it's treated as a regular expression. 101 | 102 | This example sets ``http.request`` to 200, all ``http.response!`` channels 103 | to 10, and all ``websocket.send!`` channels to 20: 104 | 105 | .. code-block:: python 106 | 107 | channel_capacity={ 108 | "http.request": 200, 109 | "http.response!*": 10, 110 | re.compile(r"^websocket.send\!.+"): 20, 111 | } 112 | 113 | If you want to enforce a matching order, use an ``OrderedDict`` as the 114 | argument; channels will then be matched in the order the dict provides them. 115 | 116 | Dependencies 117 | ------------ 118 | 119 | All Channels projects currently support Python 2.7, 3.4 and 3.5. 120 | 121 | Contributing 122 | ------------ 123 | 124 | Please refer to the 125 | `main Channels contributing docs `_. 126 | That also contains advice on how to set up the development environment and run the tests. 127 | 128 | Maintenance and Security 129 | ------------------------ 130 | 131 | To report security issues, please contact security@djangoproject.com. For GPG 132 | signatures and more security process information, see 133 | https://docs.djangoproject.com/en/dev/internals/security/. 134 | 135 | To report bugs or request new features, please open a new GitHub issue. 136 | 137 | This repository is part of the Channels project. For the shepherd and maintenance team, please see the 138 | `main Channels readme `_. 139 | -------------------------------------------------------------------------------- /asgi_ipc/core.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import msgpack 4 | import random 5 | import six 6 | import string 7 | import time 8 | from asgiref.base_layer import BaseChannelLayer 9 | 10 | from .store import ChannelMemoryStore, GroupMemoryStore 11 | 12 | 13 | class IPCChannelLayer(BaseChannelLayer): 14 | """ 15 | Posix IPC backed channel layer, using the posix_ipc module's shared memory 16 | and sempahore components. 17 | 18 | It uses mmap'd shared memory areas to store msgpack'd versions of the 19 | datastructures; see the store module for more information. 20 | """ 21 | 22 | def __init__(self, 23 | prefix="asgi", 24 | expiry=60, 25 | group_expiry=86400, 26 | capacity=10, 27 | channel_capacity=None, 28 | message_memory=100*1024*1024, 29 | group_memory=20*1024*1024, 30 | ): 31 | super(IPCChannelLayer, self).__init__( 32 | expiry=expiry, 33 | group_expiry=group_expiry, 34 | capacity=capacity, 35 | channel_capacity=channel_capacity, 36 | ) 37 | self.prefix = prefix 38 | # Table with queued messages 39 | self.message_store = ChannelMemoryStore("%s-messages" % prefix, message_memory) 40 | # Table containing all groups to flush 41 | self.group_store = GroupMemoryStore("%s-groups" % prefix, group_memory) 42 | 43 | # -------- 44 | # ASGI API 45 | # -------- 46 | 47 | extensions = ["flush", "groups"] 48 | 49 | def send(self, channel, message): 50 | # Type check 51 | assert isinstance(message, dict), "message is not a dict" 52 | assert self.valid_channel_name(channel), "channel name not valid" 53 | # Make sure the message does not contain reserved keys 54 | assert "__asgi_channel__" not in message 55 | # If it's a process-local channel, strip off local part and stick full name in message 56 | if "!" in channel: 57 | message = dict(message.items()) 58 | message['__asgi_channel__'] = channel 59 | channel = self.non_local_name(channel) 60 | # Write message into the correct message queue with a size check 61 | channel_size = self.message_store.length(channel) 62 | if channel_size >= self.get_capacity(channel): 63 | raise self.ChannelFull() 64 | else: 65 | self.message_store.append( 66 | channel, 67 | msgpack.packb(message, use_bin_type=True), 68 | time.time() + self.expiry, 69 | ) 70 | 71 | def receive(self, channels, block=False): 72 | if not channels: 73 | return None, None 74 | channels = list(channels) 75 | assert all( 76 | self.valid_channel_name(channel, receive=True) for channel in channels 77 | ), "one or more channel names invalid" 78 | random.shuffle(channels) 79 | # Try to pop off all of the named channels 80 | for channel in channels: 81 | # See if there is an unexpired message 82 | try: 83 | message = self.message_store.pop(channel) 84 | except IndexError: 85 | continue 86 | message = msgpack.unpackb(message, encoding="utf8") 87 | # If there is a full channel name stored in the message, unpack it. 88 | if "__asgi_channel__" in message: 89 | channel = message['__asgi_channel__'] 90 | del message['__asgi_channel__'] 91 | return channel, message 92 | return None, None 93 | 94 | def new_channel(self, pattern): 95 | assert isinstance(pattern, six.text_type) 96 | # Keep making channel names till one isn't present. 97 | while True: 98 | random_string = "".join(random.sample(string.ascii_letters, 12)) 99 | assert pattern.endswith("?") 100 | new_name = pattern + random_string 101 | if not self.message_store.length(new_name): 102 | return new_name 103 | else: 104 | continue 105 | 106 | # ---------------- 107 | # Groups extension 108 | # ---------------- 109 | 110 | def group_add(self, group, channel): 111 | """ 112 | Adds the channel to the named group 113 | """ 114 | assert self.valid_group_name(group), "Invalid group name" 115 | self.group_store.add(group, channel, time.time() + self.group_expiry) 116 | 117 | def group_discard(self, group, channel): 118 | """ 119 | Removes the channel from the named group if it is in the group; 120 | does nothing otherwise (does not error) 121 | """ 122 | assert self.valid_group_name(group), "Invalid group name" 123 | self.group_store.discard(group, channel) 124 | 125 | def send_group(self, group, message): 126 | """ 127 | Sends a message to the entire group. 128 | """ 129 | assert self.valid_group_name(group), "Invalid group name" 130 | for channel in self.group_channels(group): 131 | try: 132 | self.send(channel, message) 133 | except self.ChannelFull: 134 | pass 135 | 136 | def group_channels(self, group): 137 | return self.group_store.flush_expired(group) 138 | 139 | # --------------- 140 | # Flush extension 141 | # --------------- 142 | 143 | def flush(self): 144 | """ 145 | Deletes all messages and groups. 146 | """ 147 | self.message_store.flush_all() 148 | self.group_store.flush_all() 149 | 150 | def __str__(self): 151 | return "%s(prefix=%s)" % (self.__class__.__name__, self.prefix) 152 | -------------------------------------------------------------------------------- /asgi_ipc/store.py: -------------------------------------------------------------------------------- 1 | import posix_ipc 2 | import mmap 3 | import time 4 | import contextlib 5 | from six.moves import cPickle as pickle 6 | 7 | 8 | class BaseMemoryStore(object): 9 | """ 10 | Implements the base of shared memory stores, providing an mmap-ed shared 11 | memory area, a semaphore for controlling access to it, and common logic 12 | to mutate a pickled value in that memory area atomically. 13 | 14 | POSIX IPC Message Queues are not used as their default limits under most 15 | kernels are too small (8KB messages and 256 queues max); channels is a 16 | little... heavier than that. 17 | """ 18 | 19 | # The number of seconds to wait for a lock before raising an error 20 | TIMEOUT = 10 21 | 22 | # The default value's factory 23 | DEFAULT_FACTORY = dict 24 | 25 | def __init__(self, path, memory_size): 26 | self.memory_size = memory_size 27 | self.path = path 28 | self.semaphore = posix_ipc.Semaphore(path, flags=posix_ipc.O_CREAT, initial_value=1) 29 | self.memory = posix_ipc.SharedMemory(path, flags=posix_ipc.O_CREAT, size=self.memory_size) 30 | self.mmap = mmap.mmap(self.memory.fd, self.memory.size) 31 | self.deadlock_error = RuntimeError( 32 | "Semaphore appears to be deadlocked. Kill all channels processes " 33 | "and remove files named like %s in /dev/shm" % self.path 34 | ) 35 | 36 | @contextlib.contextmanager 37 | def mutate_value(self): 38 | """ 39 | Allows mutation of the value safely. 40 | """ 41 | # Get the semaphore with an emergency timeout to detect deadlock conditions 42 | try: 43 | self.semaphore.acquire(self.TIMEOUT) 44 | except posix_ipc.BusyError: 45 | raise self.deadlock_error 46 | try: 47 | # Load the value from the shared memory segment (if populated) 48 | self.mmap.seek(0) 49 | # Memory can be empty but have a length. Pickle opcodes 50 | # starts at 0x80. If we read zero, memory was not 51 | # initiated yet. 52 | if not self.mmap.read_byte(): 53 | value = self.DEFAULT_FACTORY() 54 | else: 55 | self.mmap.seek(0) 56 | try: 57 | value = pickle.load(self.mmap) 58 | except EOFError: 59 | value = self.DEFAULT_FACTORY() 60 | # Let the inside run 61 | yield value 62 | # Dump the value back into the shared memory segment 63 | self.mmap.seek(0) 64 | pickle.dump(value, self.mmap, protocol=2) 65 | finally: 66 | # Release semaphore 67 | self.semaphore.release() 68 | 69 | def get_value(self): 70 | """ 71 | Returns the value in the store safely, but does not allow safe mutation. 72 | """ 73 | # Get the semaphore with an emergency timeout to detect deadlock conditions 74 | try: 75 | self.semaphore.acquire(self.TIMEOUT) 76 | except posix_ipc.BusyError: 77 | raise self.deadlock_error 78 | try: 79 | # Load the value from the shared memory segment (if populated) 80 | self.mmap.seek(0) 81 | try: 82 | value = pickle.load(self.mmap) 83 | except EOFError: 84 | value = self.DEFAULT_FACTORY() 85 | return value 86 | finally: 87 | # Release semaphore 88 | self.semaphore.release() 89 | 90 | def flush_all(self): 91 | # Get the semaphore with an emergency timeout to detect deadlock conditions 92 | try: 93 | self.semaphore.acquire(self.TIMEOUT) 94 | except posix_ipc.BusyError: 95 | raise self.deadlock_error 96 | try: 97 | # Just write over the mmap area 98 | self.mmap.seek(0) 99 | pickle.dump(self.DEFAULT_FACTORY(), self.mmap, protocol=2) 100 | finally: 101 | # Release semaphore 102 | self.semaphore.release() 103 | 104 | 105 | class ChannelMemoryStore(BaseMemoryStore): 106 | """ 107 | Implements a shared memory store that maps unicode strings to ordered 108 | channels of binary blobs with expiry times. 109 | """ 110 | 111 | def append(self, name, item, expiry): 112 | """ 113 | Adds a binary blob to the right of the channel. 114 | """ 115 | with self.mutate_value() as value: 116 | value.setdefault(name, []).append((item, expiry)) 117 | 118 | def pop(self, name): 119 | """ 120 | Tries to pop an item off of the left of the channel, raising IndexError if 121 | no item was available to pop. 122 | """ 123 | with self.mutate_value() as value: 124 | # See if the channel even exists 125 | if name not in value: 126 | raise IndexError("Channel %s is empty" % name) 127 | # Go through the channel until we find a nonexpired message 128 | for i, item_and_expiry in enumerate(value[name]): 129 | if item_and_expiry[1] >= time.time(): 130 | value[name] = value[name][i + 1:] 131 | return item_and_expiry[0] 132 | # All messages are expired! 133 | del value[name] 134 | raise IndexError("Channel %s is empty" % name) 135 | 136 | def length(self, name): 137 | """ 138 | Removes all expired items from the channel, and returns the 139 | number of messages in the channel. 140 | """ 141 | with self.mutate_value() as value: 142 | new_contents = [ 143 | (item, expiry) 144 | for item, expiry 145 | in value.get(name, []) 146 | if expiry >= time.time() 147 | ] 148 | if new_contents: 149 | value[name] = new_contents 150 | return len(new_contents) 151 | else: 152 | if name in value: 153 | del value[name] 154 | return 0 155 | 156 | 157 | class GroupMemoryStore(BaseMemoryStore): 158 | """ 159 | Implements a shared memory store that maps unicode strings to sets 160 | of (unicode string, expiry time). 161 | """ 162 | 163 | def add(self, name, item, expiry): 164 | """ 165 | Adds a group member with an expiry time. If it already exists, 166 | update the expiry time. 167 | """ 168 | with self.mutate_value() as value: 169 | value.setdefault(name, {})[item] = expiry 170 | 171 | def discard(self, name, item): 172 | """ 173 | Removes a group member if it exists. If not, silently returns. 174 | """ 175 | with self.mutate_value() as value: 176 | if name in value and item in value[name]: 177 | del value[name][item] 178 | 179 | def flush(self, name): 180 | """ 181 | Removes all members from the group 182 | """ 183 | with self.mutate_value() as value: 184 | if name in value: 185 | del value[name] 186 | 187 | def flush_expired(self, name): 188 | """ 189 | Removes all members from the group who have expired, and returns the 190 | new list of members. 191 | """ 192 | try: 193 | with self.mutate_value() as value: 194 | value[name] = { 195 | item: expiry 196 | for item, expiry in value[name].items() 197 | if expiry >= time.time() 198 | } 199 | except KeyError: 200 | return [] 201 | return value[name].keys() 202 | 203 | --------------------------------------------------------------------------------