├── .github └── workflows │ └── codeql-analysis.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── SECURITY.md ├── __init__.py ├── artillery_test.yml ├── chat.py ├── docker-compose.yml ├── requirements.txt └── templates ├── chat.html └── moderator_chat.html /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ master ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ master ] 20 | schedule: 21 | - cron: '21 12 * * 4' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'python' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 37 | # Learn more: 38 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 39 | 40 | steps: 41 | - name: Checkout repository 42 | uses: actions/checkout@v2 43 | 44 | # Initializes the CodeQL tools for scanning. 45 | - name: Initialize CodeQL 46 | uses: github/codeql-action/init@v1 47 | with: 48 | languages: ${{ matrix.language }} 49 | # If you wish to specify custom queries, you can do so here or in a config file. 50 | # By default, queries listed here will override any specified in a config file. 51 | # Prefix the list here with "+" to use these queries and those in the config file. 52 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 53 | 54 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 55 | # If this step fails, then you should remove it and run the build manually (see below) 56 | - name: Autobuild 57 | uses: github/codeql-action/autobuild@v1 58 | 59 | # ℹ️ Command-line programs to run using the OS shell. 60 | # 📚 https://git.io/JvXDl 61 | 62 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 63 | # and modify them (or add more) to build your code if your project 64 | # uses a compiled language 65 | 66 | #- run: | 67 | # make bootstrap 68 | # make release 69 | 70 | - name: Perform CodeQL Analysis 71 | uses: github/codeql-action/analyze@v1 72 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-alpine 2 | WORKDIR /code 3 | # ENV FLASK_APP app.py 4 | RUN apk add --no-cache gcc musl-dev linux-headers make python3-dev openssl-dev libffi-dev git 5 | COPY requirements.txt requirements.txt 6 | RUN pip install -U setuptools pip 7 | # docker overwrites the src location for editable packages so we pass in a --src path that doesnt get blatted 8 | # https://stackoverflow.com/questions/29905909/pip-install-e-packages-dont-appear-in-docker 9 | RUN pip install -r requirements.txt --src /usr/local/src 10 | COPY . . 11 | CMD ["python", "chat.py"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 ludwig404 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # redis-streams-fastapi-chat 2 | A simple demo of Redis Streams backed Chat app using Websockets, Python Asyncio and FastAPI/Starlette. 3 | 4 | Requires Python version >= 3.6 and Redis 5 | 6 | # Overview 7 | This project has been created to help understand some related concepts. Python standard library asyncio, websockets (which are often cited as a classic use case for async python code), also Redis Streams. It is very much inteded to be an intentionally simple starting point rather than a usable product as is. 8 | 9 | # Installation 10 | 11 | ```shell 12 | $ pip install -r requirements.txt 13 | ``` 14 | 15 | # Usage 16 | 17 | ```shell 18 | $ python chat.py 19 | ``` 20 | 21 | # Docker compose 22 | If you don't have redis installed you can use the docker-compose.yml file to set up a 23 | working environment. 24 | 25 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | This software is in a pre-release state it is provided "as is" for learning and creating a foundation for your own project. 4 | 5 | ## Supported Versions 6 | 7 | None 8 | 9 | ## Reporting a Vulnerability 10 | 11 | If you find a securtiy issue please use github issues to inform me and any other users of this 12 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /artillery_test.yml: -------------------------------------------------------------------------------- 1 | config: 2 | target: "ws://0.0.0.0:9080/ws" 3 | ensure: 4 | maxErrorRate: 1 5 | phases: 6 | - duration: 2 7 | arrivalCount: 2 8 | name: "Warming up" 9 | - duration: 10 10 | arrivalCount: 20 11 | name: "Max load" 12 | scenarios: 13 | - engine: ws 14 | flow: 15 | - think: 6 16 | - send: 17 | - msg: "foo" -------------------------------------------------------------------------------- /chat.py: -------------------------------------------------------------------------------- 1 | import os 2 | import asyncio 3 | import aioredis 4 | import uvloop 5 | import socket 6 | import uuid 7 | import contextvars 8 | from fastapi import FastAPI, Depends, Request 9 | from starlette.staticfiles import StaticFiles 10 | from starlette.templating import Jinja2Templates 11 | from starlette.middleware.base import BaseHTTPMiddleware 12 | from starlette.websockets import WebSocket, WebSocketDisconnect 13 | 14 | from websockets.exceptions import ConnectionClosedError, ConnectionClosedOK 15 | from aioredis.errors import ConnectionClosedError as ServerConnectionClosedError 16 | 17 | REDIS_HOST = 'localhost' 18 | REDIS_PORT = 6379 19 | XREAD_TIMEOUT = 0 20 | XREAD_COUNT = 100 21 | NUM_PREVIOUS = 30 22 | STREAM_MAX_LEN = 1000 23 | ALLOWED_ROOMS = ['chat:1', 'chat:2', 'chat:3'] 24 | PORT = 9080 25 | HOST = "0.0.0.0" 26 | 27 | cvar_client_addr = contextvars.ContextVar('client_addr', default=None) 28 | cvar_chat_info = contextvars.ContextVar('chat_info', default=None) 29 | cvar_tenant = contextvars.ContextVar('tenant', default=None) 30 | cvar_redis = contextvars.ContextVar('redis', default=None) 31 | 32 | 33 | class CustomHeaderMiddleware(BaseHTTPMiddleware): 34 | def __init__(self, app, header_value='Example'): 35 | print('__init__') 36 | super().__init__(app) 37 | self.header_value = header_value 38 | 39 | async def dispatch(self, request, call_next): 40 | response = await call_next(request) 41 | response.headers['Custom'] = self.header_value 42 | return response 43 | 44 | 45 | asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) 46 | app = FastAPI() 47 | app.add_middleware(CustomHeaderMiddleware) 48 | templates = Jinja2Templates(directory="templates") 49 | 50 | 51 | def get_local_ip(): 52 | """ 53 | copy and paste from 54 | https://stackoverflow.com/questions/166506/finding-local-ip-addresses-using-pythons-stdlib 55 | """ 56 | if os.environ.get('CHAT_HOST_IP', False): 57 | return os.environ['CHAT_HOST_IP'] 58 | try: 59 | ip = [l for l in ( 60 | [ip for ip in socket.gethostbyname_ex(socket.gethostname())[2] if 61 | not ip.startswith("127.")][:1], [ 62 | [(s.connect(('8.8.8.8', 53)), s.getsockname()[0], s.close()) for s 63 | in 64 | [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]][0][1]]) if l][ 65 | 0][ 66 | 0] 67 | except OSError as e: 68 | print(e) 69 | return '127.0.0.1' 70 | 71 | return ip 72 | 73 | 74 | async def get_redis_pool(): 75 | try: 76 | pool = await aioredis.create_redis_pool( 77 | (REDIS_HOST, REDIS_PORT), encoding='utf-8') 78 | return pool 79 | except ConnectionRefusedError as e: 80 | print('cannot connect to redis on:', REDIS_HOST, REDIS_PORT) 81 | return None 82 | 83 | 84 | async def get_chat_history(): 85 | pass 86 | 87 | 88 | async def ws_send_moderator(websocket: WebSocket, chat_info: dict): 89 | """ 90 | wait for new items on chat stream and 91 | send data from server to client over a WebSocket 92 | 93 | :param websocket: 94 | :type websocket: 95 | :param chat_info: 96 | :type chat_info: 97 | """ 98 | pool = await get_redis_pool() 99 | streams = chat_info['room'].split(',') 100 | latest_ids = ['$' for i in streams] 101 | ws_connected = True 102 | print(streams, latest_ids) 103 | while pool and ws_connected: 104 | try: 105 | events = await pool.xread( 106 | streams=streams, 107 | count=XREAD_COUNT, 108 | timeout=XREAD_TIMEOUT, 109 | latest_ids=latest_ids 110 | ) 111 | for _, e_id, e in events: 112 | e['e_id'] = e_id 113 | await websocket.send_json(e) 114 | #latest_ids = [e_id] 115 | except ConnectionClosedError: 116 | ws_connected = False 117 | 118 | except ConnectionClosedOK: 119 | ws_connected = False 120 | 121 | 122 | async def ws_send(websocket: WebSocket, chat_info: dict): 123 | """ 124 | wait for new items on chat stream and 125 | send data from server to client over a WebSocket 126 | 127 | :param websocket: 128 | :type websocket: 129 | :param chat_info: 130 | :type chat_info: 131 | """ 132 | pool = await get_redis_pool() 133 | latest_ids = ['$'] 134 | ws_connected = True 135 | first_run = True 136 | while pool and ws_connected: 137 | try: 138 | if first_run: 139 | # fetch some previous chat history 140 | events = await pool.xrevrange( 141 | stream=cvar_tenant.get() + ":stream", 142 | count=NUM_PREVIOUS, 143 | start='+', 144 | stop='-' 145 | ) 146 | first_run = False 147 | events.reverse() 148 | for e_id, e in events: 149 | e['e_id'] = e_id 150 | await websocket.send_json(e) 151 | else: 152 | events = await pool.xread( 153 | streams=[cvar_tenant.get() + ":stream"], 154 | count=XREAD_COUNT, 155 | timeout=XREAD_TIMEOUT, 156 | latest_ids=latest_ids 157 | ) 158 | for _, e_id, e in events: 159 | e['e_id'] = e_id 160 | await websocket.send_json(e) 161 | latest_ids = [e_id] 162 | #print('################contextvar ', cvar_tenant.get()) 163 | except ConnectionClosedError: 164 | ws_connected = False 165 | 166 | except ConnectionClosedOK: 167 | ws_connected = False 168 | 169 | except ServerConnectionClosedError: 170 | print('redis server connection closed') 171 | return 172 | pool.close() 173 | 174 | 175 | async def ws_recieve(websocket: WebSocket, chat_info: dict): 176 | """ 177 | receive json data from client over a WebSocket, add messages onto the 178 | associated chat stream 179 | 180 | :param websocket: 181 | :type websocket: 182 | :param chat_info: 183 | :type chat_info: 184 | """ 185 | 186 | ws_connected = False 187 | pool = await get_redis_pool() 188 | added = await add_room_user(chat_info, pool) 189 | 190 | if added: 191 | await announce(pool, chat_info, 'connected') 192 | ws_connected = True 193 | else: 194 | print('duplicate user error') 195 | 196 | while ws_connected: 197 | try: 198 | data = await websocket.receive_json() 199 | #print(data) 200 | if type(data) == list and len(data): 201 | data = data[0] 202 | fields = { 203 | 'uname': chat_info['username'], 204 | 'msg': data['msg'], 205 | 'type': 'comment', 206 | 'room': chat_info['room'] 207 | } 208 | await pool.xadd(stream=cvar_tenant.get() + ":stream", 209 | fields=fields, 210 | message_id=b'*', 211 | max_len=STREAM_MAX_LEN) 212 | #print('################contextvar ', cvar_tenant.get()) 213 | except WebSocketDisconnect: 214 | await remove_room_user(chat_info, pool) 215 | await announce(pool, chat_info, 'disconnected') 216 | ws_connected = False 217 | 218 | except ServerConnectionClosedError: 219 | print('redis server connection closed') 220 | return 221 | 222 | except ConnectionRefusedError: 223 | print('redis server connection closed') 224 | return 225 | 226 | pool.close() 227 | 228 | 229 | async def add_room_user(chat_info: dict, pool): 230 | #added = await pool.sadd(chat_info['room']+":users", chat_info['username']) 231 | added = await pool.sadd(cvar_tenant.get()+":users", cvar_chat_info.get()['username']) 232 | return added 233 | 234 | 235 | async def remove_room_user(chat_info: dict, pool): 236 | #removed = await pool.srem(chat_info['room']+":users", chat_info['username']) 237 | removed = await pool.srem(cvar_tenant.get()+":users", cvar_chat_info.get()['username']) 238 | return removed 239 | 240 | 241 | async def room_users(chat_info: dict, pool): 242 | #users = await pool.smembers(chat_info['room']+":users") 243 | users = await pool.smembers(cvar_tenant.get()+":users") 244 | print(len(users)) 245 | return users 246 | 247 | 248 | async def announce(pool, chat_info: dict, action: str): 249 | """ 250 | add an announcement event onto the redis chat stream 251 | """ 252 | users = await room_users(chat_info, pool) 253 | fields = { 254 | 'msg': f"{chat_info['username']} {action}", 255 | 'action': action, 256 | 'type': 'announcement', 257 | 'users': ", ".join(users), 258 | 'room': chat_info['room'] 259 | } 260 | #print(fields) 261 | 262 | await pool.xadd(stream=cvar_tenant.get() + ":stream", 263 | fields=fields, 264 | message_id=b'*', 265 | max_len=STREAM_MAX_LEN) 266 | 267 | 268 | async def chat_info_vars(username: str = None, room: str = None): 269 | """ 270 | URL parameter info needed for a user to participate in a chat 271 | :param username: 272 | :type username: 273 | :param room: 274 | :type room: 275 | """ 276 | if username is None and room is None: 277 | return {"username": str(uuid.uuid4()), "room": 'chat:1'} 278 | return {"username": username, "room": room} 279 | 280 | 281 | @app.websocket("/ws") 282 | async def websocket_endpoint(websocket: WebSocket, 283 | chat_info: dict = Depends(chat_info_vars)): 284 | #print('request.hostname', websocket.url.hostname) 285 | tenant_id = ":".join([websocket.url.hostname.replace('.', '_'), 286 | chat_info['room']]) 287 | cvar_tenant.set(tenant_id) 288 | cvar_chat_info.set(chat_info) 289 | 290 | 291 | # check the user is allowed into the chat room 292 | verified = await verify_user_for_room(chat_info) 293 | # open connection 294 | await websocket.accept() 295 | if not verified: 296 | 297 | print('failed verification') 298 | print(chat_info) 299 | await websocket.close() 300 | else: 301 | 302 | # spin up coro's for inbound and outbound communication over the socket 303 | await asyncio.gather(ws_recieve(websocket, chat_info), 304 | ws_send(websocket, chat_info)) 305 | 306 | 307 | @app.websocket("/ws/moderator") 308 | async def websocket_moderator_endpoint(websocket: WebSocket, 309 | chat_info: dict = Depends(chat_info_vars)): 310 | # check the user is allowed into the chat room 311 | # verified = await verify_user_for_room(chat_info) 312 | # open connection 313 | 314 | if not chat_info['username'] == 'moderator': 315 | print('failed verification') 316 | await websocket.close() 317 | return 318 | 319 | await websocket.accept() 320 | # spin up coro's for inbound and outbound communication over the socket 321 | await asyncio.gather(ws_send_moderator(websocket, chat_info)) 322 | 323 | 324 | @app.get("/") 325 | async def get(request: Request): 326 | return templates.TemplateResponse("chat.html", 327 | {"request": request, 328 | "ip": get_local_ip(), 329 | "port": PORT}) 330 | 331 | 332 | @app.get("/moderator") 333 | async def get(request: Request): 334 | return templates.TemplateResponse("moderator_chat.html", 335 | {"request": request, 336 | "ip": get_local_ip(), 337 | "port": PORT}) 338 | 339 | 340 | async def verify_user_for_room(chat_info): 341 | verified = True 342 | pool = await get_redis_pool() 343 | if not pool: 344 | print('Redis connection failure') 345 | return False 346 | # check for duplicated user names 347 | already_exists = await pool.sismember(cvar_tenant.get()+":users", cvar_chat_info.get()['username']) 348 | if already_exists: 349 | print(chat_info['username'] +' user already_exists in ' + chat_info['room']) 350 | verified = False 351 | # check for restricted names 352 | 353 | # check for restricted rooms 354 | # check for non existent rooms 355 | # whitelist rooms 356 | if not chat_info['room'] in ALLOWED_ROOMS: 357 | verified = False 358 | pool.close() 359 | return verified 360 | 361 | 362 | @app.on_event("startup") 363 | async def handle_startup(): 364 | try: 365 | pool = await aioredis.create_redis_pool( 366 | (REDIS_HOST, REDIS_PORT), encoding='utf-8', maxsize=20) 367 | cvar_redis.set(pool) 368 | print("Connected to Redis on ", REDIS_HOST, REDIS_PORT) 369 | except ConnectionRefusedError as e: 370 | print('cannot connect to redis on:', REDIS_HOST, REDIS_PORT) 371 | return 372 | 373 | 374 | @app.on_event("shutdown") 375 | async def handle_shutdown(): 376 | redis = cvar_redis.get() 377 | redis.close() 378 | await redis.wait_closed() 379 | print("closed connection Redis on ", REDIS_HOST, REDIS_PORT) 380 | 381 | 382 | if __name__ == "__main__": 383 | import uvicorn 384 | print(dir(app)) 385 | print(app.url_path_for('websocket_endpoint')) 386 | uvicorn.run('chat:app', host=HOST, port=PORT, log_level='info', reload=True)#, uds='uvicorn.sock') 387 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.4' 2 | services: 3 | chat: 4 | build: . 5 | ports: 6 | - "9080:9080" 7 | - "8081:8081" 8 | - "8082:8082" 9 | volumes: 10 | - .:/code 11 | links: 12 | - "redis" 13 | redis: 14 | image: "redis:6.0-rc2-alpine3.11" 15 | ports: 16 | - 6379:6379 17 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | uvicorn==0.12.1 2 | websockets==9.1 3 | fastapi==0.65.2 4 | aioredis==1.3.1 5 | redis==4.5.4 6 | uvloop==0.15.2 7 | jinja2==2.11.3 8 | aiofiles==0.6.0 9 | httpx==0.23.0 10 | itsdangerous==1.1.0 11 | databases[sqlite]==0.4.3 12 | sqlalchemy==1.3.0 -------------------------------------------------------------------------------- /templates/chat.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |
4 |