├── consistency_isolation ├── snapshot │ ├── __init__.py │ └── snapshot_db.py ├── no_isolation │ ├── __init__.py │ └── db.py ├── read_committed │ ├── __init__.py │ └── rc_db.py ├── serializable │ ├── __init__.py │ └── serializable_db.py ├── take_shuttle.py └── __init__.py └── README.md /consistency_isolation/snapshot/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /consistency_isolation/no_isolation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /consistency_isolation/read_committed/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /consistency_isolation/serializable/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /consistency_isolation/no_isolation/db.py: -------------------------------------------------------------------------------- 1 | from .. import Server, ServerConnection 2 | 3 | db: dict[str, str] = {} 4 | 5 | 6 | def server_thread(server: ServerConnection): 7 | while True: 8 | cmd = server.next_command() 9 | if cmd is None or cmd.name == "bye": 10 | break 11 | elif cmd.name == "set": 12 | db[cmd.key] = cmd.value 13 | elif cmd.name == "get": 14 | try: 15 | server.send(db[cmd.key]) 16 | except KeyError: 17 | server.send("not found") 18 | elif cmd.name == "commit": 19 | pass 20 | else: 21 | server.send("invalid syntax") 22 | 23 | 24 | Server().serve(server_thread) 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Consistency and Isolation for Python Programmers 2 | 3 | Files related to A. Jesse Jiryu Davis's PyCon 2023 talk. [Blog post with links to video and a podcast interview](https://emptysqua.re/blog/pycon-2023-consistency-isolation/). 4 | 5 | Requires Python 3.10+. 6 | 7 | Run the server like: `python no_isolation/db.py`. (Substitute whichever variant of the server you 8 | want to run.) 9 | 10 | Interact with the server using Telnet: 11 | ``` 12 | > brew install telnet 13 | > telnet localhost 14 | Trying 127.0.0.1... 15 | Connected to localhost. 16 | Escape character is '^]'. 17 | Welcome to no_isolation/db.py! 18 | Type 'bye' to quit. 19 | db> get foo 20 | not found 21 | db> set foo 1 22 | db> get foo 23 | 1 24 | db> bye 25 | Connection closed by foreign host. 26 | ``` 27 | -------------------------------------------------------------------------------- /consistency_isolation/read_committed/rc_db.py: -------------------------------------------------------------------------------- 1 | import collections 2 | from threading import RLock 3 | 4 | from .. import Server, ServerConnection 5 | 6 | db: dict[str, str] = {} 7 | locks: dict[str, RLock] = collections.defaultdict(RLock) 8 | 9 | def server_thread(server: ServerConnection): 10 | txn_locks = [] 11 | 12 | def acquire_lock(k: str): 13 | lock = locks[k]; lock.acquire() 14 | txn_locks.append(lock) 15 | 16 | while True: 17 | cmd = server.next_command() 18 | if cmd is None or cmd.name == "bye": 19 | break 20 | elif cmd.name == "set": 21 | acquire_lock(cmd.key) 22 | db[cmd.key] = cmd.value 23 | elif cmd.name == "get": 24 | key = cmd.key 25 | with locks[key]: 26 | try: 27 | server.send(db[key]) 28 | except KeyError: 29 | server.send("not found") 30 | elif cmd.name == "commit": 31 | while txn_locks: 32 | txn_locks.pop().release() 33 | else: 34 | server.send("invalid syntax") 35 | 36 | 37 | Server().serve(server_thread) 38 | -------------------------------------------------------------------------------- /consistency_isolation/serializable/serializable_db.py: -------------------------------------------------------------------------------- 1 | import collections 2 | from threading import RLock 3 | 4 | from .. import Server, ServerConnection 5 | 6 | db: dict[str, str] = {} 7 | locks: dict[str, RLock] = collections.defaultdict(RLock) 8 | 9 | 10 | def server_thread(server: ServerConnection): 11 | txn_locks = [] 12 | 13 | def acquire_lock(k: str): 14 | lock = locks[k] 15 | lock.acquire() 16 | txn_locks.append(lock) 17 | 18 | while True: 19 | cmd = server.next_command() 20 | if cmd is None or cmd.name == "bye": 21 | break 22 | elif cmd.name == "set": 23 | acquire_lock(cmd.key) 24 | db[cmd.key] = cmd.value 25 | elif cmd.name == "get": 26 | acquire_lock(cmd.key) 27 | try: 28 | server.send(db[cmd.key]) 29 | except KeyError: 30 | # Keep the lock to prevent phantoms. 31 | server.send("not found") 32 | elif cmd.name == "commit": 33 | while txn_locks: 34 | txn_locks.pop().release() 35 | 36 | # TODO: roll back the transaction and drop locks. 37 | 38 | 39 | Server().serve(server_thread) 40 | -------------------------------------------------------------------------------- /consistency_isolation/take_shuttle.py: -------------------------------------------------------------------------------- 1 | import threading 2 | 3 | from . import ClientConnection 4 | 5 | lock = threading.Lock() 6 | n_taken = 0 7 | key = "number-of-shuttles" 8 | client = ClientConnection() 9 | client.send(f"set {key} 1") # There is one shuttle in the shuttlebay. 10 | client.send("commit") 11 | 12 | 13 | def withdraw_1(thread_id): 14 | global n_taken 15 | c = ClientConnection() 16 | c.send(f"get {key}") 17 | n = int(c.readline()) 18 | print(f"Thread {thread_id} sees: {n}") 19 | if n >= 1: 20 | n -= 1 21 | c.send(f"set {key} {n}") 22 | print(f"Thread {thread_id} took the shuttle") 23 | with lock: 24 | n_taken += 1 25 | else: 26 | print(f"Thread {thread_id} can't take the shuttle") 27 | 28 | c.send("commit") 29 | 30 | 31 | t1 = threading.Thread(target=withdraw_1, args=[1]) 32 | t2 = threading.Thread(target=withdraw_1, args=[2]) 33 | t1.start() 34 | t2.start() 35 | t1.join() 36 | t2.join() 37 | print(f"Number of shuttles taken: {n_taken}") 38 | # In snapshot isolation, we need to restart txn to see fresh data. 39 | client = ClientConnection() 40 | client.send(f"get {key}") 41 | print(f"Number in the shuttlebay according to the server: {client.readline()}") 42 | client.send("commit") 43 | -------------------------------------------------------------------------------- /consistency_isolation/snapshot/snapshot_db.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import threading 3 | 4 | from .. import Server, ServerConnection 5 | 6 | db = {} 7 | write_time = collections.defaultdict(int) 8 | next_time = 1 9 | lock = threading.Lock() # Guards db, write_time, and next_time. 10 | 11 | def server_thread(server: ServerConnection): 12 | global next_time 13 | with lock: 14 | snapshot = db.copy(); writes = {}; start = next_time; next_time += 1 15 | 16 | while True: 17 | cmd = server.next_command() 18 | if cmd is None or cmd.name == "bye": 19 | break 20 | elif cmd.name == "set": 21 | writes[cmd.key] = snapshot[cmd.key] = cmd.value # Store writes in txn dict. 22 | elif cmd.name == "get": 23 | try: 24 | server.send(snapshot[cmd.key]) 25 | except KeyError: 26 | server.send("not found") 27 | elif cmd.name == "commit": 28 | with lock: 29 | commit_time = next_time; next_time += 1 30 | # Another txn wrote to the same key(s) and committed. 31 | if any(start <= write_time[k] <= commit_time for k in writes): 32 | server.send("aborted") 33 | else: 34 | write_time.update({k: commit_time for k in writes}) 35 | db.update(writes) 36 | snapshot = db.copy(); writes = {}; start = next_time; next_time += 1 37 | 38 | 39 | Server().serve(server_thread) 40 | -------------------------------------------------------------------------------- /consistency_isolation/__init__.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import os.path 3 | import re 4 | import socket 5 | import sys 6 | import telnetlib 7 | import threading 8 | from collections.abc import Callable 9 | 10 | 11 | class ClientConnection: 12 | def __init__(self): 13 | self._telnet: telnetlib.Telnet | None = None 14 | 15 | def readline(self) -> str: 16 | self._ensure_connected() 17 | return self._telnet.read_until("\r\n".encode()).decode().strip() 18 | 19 | def send(self, msg: str) -> None: 20 | self._ensure_connected() 21 | self._telnet.read_until("db> ".encode()) # Await prompt. 22 | self._telnet.write(msg.encode() + b"\r\n") 23 | 24 | def _ensure_connected(self) -> None: 25 | if self._telnet is None: 26 | self._telnet = telnetlib.Telnet("localhost") 27 | 28 | 29 | @dataclasses.dataclass 30 | class Command: 31 | name: str 32 | key: None | str = None 33 | value: None | str = None 34 | 35 | 36 | class ServerConnection: 37 | def __init__(self, sock: socket.socket): 38 | self._telnet = telnetlib.Telnet() 39 | self._telnet.sock = sock 40 | self._client_addr = sock.getpeername() 41 | print(f"{self._client_addr} connected") 42 | program = f"{os.path.basename(os.path.dirname(sys.argv[0]))}" \ 43 | f"/{os.path.basename(sys.argv[0])}" 44 | self.send(f"Welcome to {program}!\nType 'bye' to quit.") 45 | 46 | def readline(self) -> str | None: 47 | try: 48 | self._telnet.write("db> ".encode()) 49 | return self._telnet.read_until("\r\n".encode()).decode().strip() 50 | except (EOFError, ConnectionResetError): 51 | print(f"{self._client_addr} disconnected") 52 | self._telnet.sock.close() 53 | return None 54 | 55 | def next_command(self) -> Command | None: 56 | while True: 57 | msg = self.readline() 58 | if msg is None: 59 | return None 60 | elif msg in ("commit", "bye"): 61 | return Command(msg) 62 | elif match := re.match( 63 | r"(?Pget|set)\s+(?P\S+)(\s+(?P\S+))?", 64 | msg): 65 | return Command(**match.groupdict()) 66 | 67 | self.send("invalid syntax") 68 | # Loop around and try again. 69 | 70 | def send(self, msg: str) -> None: 71 | self._telnet.write(msg.encode() + b"\r\n") 72 | 73 | def close(self): 74 | self._telnet.sock.close() 75 | 76 | 77 | class Server: 78 | def __init__(self): 79 | self._sock: socket.socket | None = None 80 | 81 | def serve(self, worker_fn: Callable[[ServerConnection], None]): 82 | self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 83 | self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 84 | self._sock.bind(('', 23), ) 85 | self._sock.listen(10) 86 | 87 | while True: 88 | sock, addr = self._sock.accept() 89 | server = ServerConnection(sock) 90 | 91 | def work_then_close(sc: ServerConnection): 92 | try: 93 | worker_fn(sc) 94 | finally: 95 | sc.close() 96 | 97 | threading.Thread(target=work_then_close, args=(server,)).start() 98 | --------------------------------------------------------------------------------