├── .gitignore ├── Chapter01 └── 01_safe-code.py ├── Chapter02 ├── 01_threading-start.py ├── 02_threading-daemon.py ├── 03_multithreading-worker.py ├── 04_multithreading-process.py ├── 06_multiprocessing-worker.py ├── 07_futures-threads-worker.py ├── 09_futures-process-worker.py ├── 10_class_ProcessPoolExecutor.py ├── 12_class_ThreadPoolExecutor.py ├── 13_futurist-threads-worker.py ├── 15_futurist-check-and-reject.py ├── 16_futurist-periodics.py ├── 18_cotyledon.py ├── 19_producer-consumer.py └── 20_reconfig-process.py ├── Chapter03 ├── 01_blocking-socket.py ├── 02_non-blocking-socket.py ├── 03_select-select.py ├── 04_hello-asyncio.py ├── 06_asyncio-coroutine.py ├── 08_asyncio-sleep-gather.py ├── 09_aiohttp-example.py ├── 10_call-later.py ├── 11_call-later-repeatedly.py ├── 12_asyncio-tcp-server.py ├── 14_asyncio-tcp-client.py └── 15_asyncio-server-stats.py ├── Chapter05 ├── 01_push-job.py ├── 02_push-job-option.py ├── 03_celery-task.py ├── 04_celery-task-retry.py ├── 05_celery-chain.py └── 06_celery-task-queue.py ├── Chapter06 ├── 01_retrying.py ├── 02_retrying-sleep.py ├── 03_retrying-backoff.py ├── 04_retrying-tenacity.py ├── 05_tenacity-fixed-waiting.py ├── 06_tenacity-backoff.py ├── 07_tenacity-combine-wait.py ├── 08_tenacity-specific-condition.py ├── 09_tenacity-combine-condition.py └── 10_tenacity-without-decorator.py ├── Chapter07 ├── 01_threading-lock.py ├── 02_threading-reentrant-lock.py ├── 03_threading-event.py ├── 04_multiprocessing-without-lock.py ├── 05_multiprocessing-lock.py ├── 06_fasteners-interprocess-lock.py ├── 07_fasteners-decorator.py ├── 08_etcd-lock.py ├── 09_etcd-lock-with.py ├── 10_etcd-cotyledon.py ├── 11_tooz-coodinator.py ├── 12_tooz-coodinator-lock.py └── 13_tooz-with.py ├── Chapter08 ├── 01_join-group.py ├── 02_join-group-capabilities.py ├── 03_join-watchers.py ├── 05_tooz-hash-ring.py ├── 06_tooz-join-partitioned-group.py └── 07_spread-fetching-web.py ├── Chapter09 ├── 01_WSGI-basic.py ├── 02_WSGI-application.py ├── 04_publish-command.py ├── 06_message-listen.py ├── 08_flask-streamer.py ├── 11_flask-etag.py ├── 12_flask-etag-put.py ├── 13_flask-etag-retry.py ├── 14_flask-async-job.py ├── 15_requests-session.py ├── 16_requests-connection-pool.py ├── 17_requests-futures.py ├── 18_requests-futures_exam.py ├── 19_aiohttp_exam.py ├── 20_requests-comparison.py ├── 22_requests-streaming.py ├── 23_aiohttp-streaming.py ├── 25_gabbi-flask.py └── gabbi-flask.yaml ├── Chapter11 ├── 01_tox_shell │ ├── setup-memcached.sh │ ├── setup.py │ ├── test_memcached.py │ └── tox.ini ├── 02_tox_pifpaf │ ├── setup.py │ ├── test_memcached_pifpaf.py │ └── tox.ini ├── 05_memcached-fixtures.py ├── 06_app-memcached-fixtures.py └── 07_app-memcached-fixtures-all.py ├── Chapter12 ├── 03_cache-webpage.py ├── 07_memcached-connect.py ├── 08_memcached-missing-key.py ├── 09_pymemcache-fallback.py ├── 10_memcached_user_count.py └── 11_memcached-CAS.py ├── Chapter13 └── 03_memory-view-copy.py ├── README.md └── cover.jpg /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | 103 | test.py 104 | 105 | .vscode 106 | Include/ 107 | Scripts/ 108 | pip-selfcheck.json 109 | tcl/ 110 | bin/ 111 | include/ 112 | man/ -------------------------------------------------------------------------------- /Chapter01/01_safe-code.py: -------------------------------------------------------------------------------- 1 | import threading 2 | 3 | x = [] 4 | 5 | def append_two(l): 6 | l.append(2) 7 | 8 | threading.Thread(target=append_two, args=(x,)).start() 9 | 10 | x.append(1) 11 | print(x) 12 | -------------------------------------------------------------------------------- /Chapter02/01_threading-start.py: -------------------------------------------------------------------------------- 1 | import threading 2 | 3 | 4 | def print_something(something): 5 | print(something) 6 | 7 | 8 | t = threading.Thread(target=print_something, args=("hello",)) 9 | t.start() 10 | print("thread started") 11 | t.join() 12 | -------------------------------------------------------------------------------- /Chapter02/02_threading-daemon.py: -------------------------------------------------------------------------------- 1 | import threading 2 | 3 | 4 | def print_something(something): 5 | print(something) 6 | 7 | 8 | t = threading.Thread(target=print_something, args=("hello",)) 9 | t.daemon = True 10 | t.start() 11 | print("thread started") -------------------------------------------------------------------------------- /Chapter02/03_multithreading-worker.py: -------------------------------------------------------------------------------- 1 | import random 2 | import threading 3 | 4 | results = [] 5 | 6 | 7 | def compute(): 8 | results.append(sum( 9 | [random.randint(1, 100) for i in range(1000000)])) 10 | 11 | 12 | workers = [threading.Thread(target=compute) for x in range(8)] 13 | for worker in workers: 14 | worker.start() 15 | for worker in workers: 16 | worker.join() 17 | print("Results: %s" % results) -------------------------------------------------------------------------------- /Chapter02/04_multithreading-process.py: -------------------------------------------------------------------------------- 1 | import random 2 | import multiprocessing 3 | 4 | 5 | def compute(results): 6 | results.append(sum( 7 | [random.randint(1, 100) for i in range(1000000)])) 8 | 9 | 10 | with multiprocessing.Manager() as manager: 11 | results = manager.list() 12 | workers = [multiprocessing.Process(target=compute, args=(results,)) 13 | for x in range(8)] 14 | for worker in workers: 15 | worker.start() 16 | for worker in workers: 17 | worker.join() 18 | print("Results: %s" % results) -------------------------------------------------------------------------------- /Chapter02/06_multiprocessing-worker.py: -------------------------------------------------------------------------------- 1 | import multiprocessing 2 | import random 3 | 4 | 5 | def compute(n): 6 | return sum( 7 | [random.randint(1, 100) for i in range(1000000)]) 8 | 9 | 10 | # 8개의 워커를 시작한다. 11 | pool = multiprocessing.Pool(processes=8) 12 | print("Results: %s" % pool.map(compute, range(8))) -------------------------------------------------------------------------------- /Chapter02/07_futures-threads-worker.py: -------------------------------------------------------------------------------- 1 | from concurrent import futures 2 | import random 3 | 4 | 5 | def compute(): 6 | return sum( 7 | [random.randint(1, 100) for i in range(1000000)]) 8 | 9 | 10 | with futures.ThreadPoolExecutor(max_workers=8) as executor: 11 | futures = [executor.submit(compute) for _ in range(8)] 12 | 13 | results = [f.result() for f in futures] 14 | 15 | print("Results: %s" % results) -------------------------------------------------------------------------------- /Chapter02/09_futures-process-worker.py: -------------------------------------------------------------------------------- 1 | from concurrent import futures 2 | import random 3 | 4 | 5 | def compute(): 6 | return sum( 7 | [random.randint(1, 100) for i in range(1000000)]) 8 | 9 | 10 | with futures.ProcessPoolExecutor() as executor: 11 | futures = [executor.submit(compute) for _ in range(8)] 12 | 13 | results = [f.result() for f in futures] 14 | 15 | print("Results: %s" % results) -------------------------------------------------------------------------------- /Chapter02/10_class_ProcessPoolExecutor.py: -------------------------------------------------------------------------------- 1 | class ProcessPoolExecutor(_base.Executor): 2 | def __init__(self, max_workers=None): 3 | # [...] 4 | if max_workers is None: 5 | self._max_workers = multiprocessing.cpu_count() 6 | else: 7 | self._max_workers = max_workers -------------------------------------------------------------------------------- /Chapter02/12_class_ThreadPoolExecutor.py: -------------------------------------------------------------------------------- 1 | class ThreadPoolExecutor(_base.Executor): 2 | def submit(self, fn, *args, **kwargs): 3 | [...] 4 | self._adjust_thread_count() 5 | 6 | def _adjust_thread_count(self): 7 | [...] 8 | # TODO(bquinlan): Should avoid creating new threads if there are more 9 | # idle threads than items in the work queue. 10 | if len(self._threads) < self._max_workers: 11 | t = threading.Thread(target=_worker, 12 | args=(weakref.ref(self, weakref_cb), 13 | self._work_queue)) 14 | t.daemon = True 15 | t.start() 16 | self._threads.add(t) 17 | _threads_queues[t] = self._work_queue -------------------------------------------------------------------------------- /Chapter02/13_futurist-threads-worker.py: -------------------------------------------------------------------------------- 1 | import futurist 2 | from futurist import waiters 3 | import random 4 | 5 | 6 | def compute(): 7 | return sum( 8 | [random.randint(1, 100) for i in range(10000)]) 9 | 10 | 11 | with futurist.ThreadPoolExecutor(max_workers=8) as executor: 12 | futures = [executor.submit(compute) for _ in range(8)] 13 | print(executor.statistics) 14 | 15 | results = waiters.wait_for_all(futures) 16 | print(executor.statistics) 17 | 18 | print("Results: %s" % [r.result() for r in results.done]) -------------------------------------------------------------------------------- /Chapter02/15_futurist-check-and-reject.py: -------------------------------------------------------------------------------- 1 | import futurist 2 | from futurist import rejection 3 | import random 4 | 5 | 6 | def compute(): 7 | return sum( 8 | [random.randint(1, 100) for i in range(1000000)]) 9 | 10 | 11 | with futurist.ThreadPoolExecutor( 12 | max_workers=8, 13 | check_and_reject=rejection.reject_when_reached(16)) as executor: 14 | futures = [executor.submit(compute) for _ in range(20)] 15 | print(executor.statistics) 16 | 17 | results = [f.result() for f in futures] 18 | print(executor.statistics) 19 | 20 | print("Results: %s" % results) -------------------------------------------------------------------------------- /Chapter02/16_futurist-periodics.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from futurist import periodics 4 | 5 | 6 | @periodics.periodic(1) 7 | def every_one(started_at): 8 | print("1: %s" % (time.time() - started_at)) 9 | 10 | 11 | w = periodics.PeriodicWorker([ 12 | (every_one, (time.time(),), {}), 13 | ]) 14 | 15 | 16 | @periodics.periodic(4) 17 | def print_stats(): 18 | print("stats: %s" % list(w.iter_watchers())) 19 | 20 | 21 | w.add(print_stats) 22 | w.start() 23 | -------------------------------------------------------------------------------- /Chapter02/18_cotyledon.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import time 3 | 4 | import cotyledon 5 | 6 | 7 | class PrinterService(cotyledon.Service): 8 | name = "printer" 9 | 10 | def __init__(self, worker_id): 11 | super(PrinterService, self).__init__(worker_id) 12 | self._shutdown = threading.Event() 13 | 14 | def run(self): 15 | while not self._shutdown.is_set(): 16 | print("Doing stuff") 17 | time.sleep(1) 18 | 19 | def terminate(self): 20 | self._shutdown.set() 21 | 22 | 23 | # manager 생성 24 | manager = cotyledon.ServiceManager() 25 | # 2개의 PrinterService를 실행하기 위해 추가 26 | manager.add(PrinterService, 2) 27 | # manager에 추가된 작업을 모두 실행 28 | manager.run() -------------------------------------------------------------------------------- /Chapter02/19_producer-consumer.py: -------------------------------------------------------------------------------- 1 | import multiprocessing 2 | import time 3 | 4 | import cotyledon 5 | 6 | 7 | class Manager(cotyledon.ServiceManager): 8 | def __init__(self): 9 | super(Manager, self).__init__() 10 | queue = multiprocessing.Manager().Queue() 11 | self.add(ProducerService, args=(queue,)) 12 | self.add(PrinterService, args=(queue,), workers=2) 13 | 14 | 15 | class ProducerService(cotyledon.Service): 16 | def __init__(self, worker_id, queue): 17 | super(ProducerService, self).__init__(worker_id) 18 | self.queue = queue 19 | 20 | def run(self): 21 | i = 0 22 | while True: 23 | self.queue.put(i) 24 | i += 1 25 | time.sleep(1) 26 | 27 | 28 | class PrinterService(cotyledon.Service): 29 | name = "printer" 30 | 31 | def __init__(self, worker_id, queue): 32 | super(PrinterService, self).__init__(worker_id) 33 | self.queue = queue 34 | 35 | def run(self): 36 | while True: 37 | job = self.queue.get(block=True) 38 | print("I am Worker: %d PID: %d and I print %s" 39 | % (self.worker_id, self.pid, job)) 40 | 41 | 42 | Manager().run() 43 | -------------------------------------------------------------------------------- /Chapter02/20_reconfig-process.py: -------------------------------------------------------------------------------- 1 | import multiprocessing 2 | import time 3 | 4 | import cotyledon 5 | 6 | 7 | class Manager(cotyledon.ServiceManager): 8 | def __init__(self): 9 | super(Manager, self).__init__() 10 | queue = multiprocessing.Manager().Queue() 11 | self.add(ProducerService, args=(queue,)) 12 | self.printer = self.add(PrinterService, args=(queue,), workers=2) 13 | self.register_hooks(on_reload=self.reload) 14 | 15 | def reload(self): 16 | print("Reloading") 17 | self.reconfigure(self.printer, 5) 18 | 19 | 20 | class ProducerService(cotyledon.Service): 21 | def __init__(self, worker_id, queue): 22 | super(ProducerService, self).__init__(worker_id) 23 | self.queue = queue 24 | 25 | def run(self): 26 | i = 0 27 | while True: 28 | self.queue.put(i) 29 | i += 1 30 | time.sleep(1) 31 | 32 | 33 | class PrinterService(cotyledon.Service): 34 | name = "printer" 35 | 36 | def __init__(self, worker_id, queue): 37 | super(PrinterService, self).__init__(worker_id) 38 | self.queue = queue 39 | 40 | def run(self): 41 | while True: 42 | job = self.queue.get(block=True) 43 | print("I am Worker: %d PID: %d and I print %s" 44 | % (self.worker_id, self.pid, job)) 45 | 46 | 47 | Manager().run() 48 | -------------------------------------------------------------------------------- /Chapter03/01_blocking-socket.py: -------------------------------------------------------------------------------- 1 | import socket 2 | 3 | s = socket.create_connection(("httpbin.org", 80)) 4 | s.send(b"GET /delay/5 HTTP/1.1\r\nHost: httpbin.org\r\n\r\n") 5 | buf = s.recv(1024) 6 | print(buf) -------------------------------------------------------------------------------- /Chapter03/02_non-blocking-socket.py: -------------------------------------------------------------------------------- 1 | import socket 2 | 3 | s = socket.create_connection(("httpbin.org", 80)) 4 | s.setblocking(False) 5 | s.send(b"GET /delay/5 HTTP/1.1\r\nHost: httpbin.org\r\n\r\n") 6 | buf = s.recv(1024) 7 | print(buf) -------------------------------------------------------------------------------- /Chapter03/03_select-select.py: -------------------------------------------------------------------------------- 1 | import select 2 | import socket 3 | 4 | s = socket.create_connection(("httpbin.org", 80)) 5 | s.setblocking(False) 6 | s.send(b"GET /delay/1 HTTP/1.1\r\nHost: httpbin.org\r\n\r\n") 7 | while True: 8 | ready_to_read, ready_to_write, in_error = select.select( 9 | [s], [], []) 10 | if s in ready_to_read: 11 | buf = s.recv(1024) 12 | print(buf) 13 | break -------------------------------------------------------------------------------- /Chapter03/04_hello-asyncio.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def hello_world(): 5 | print("hello world!") 6 | return 42 7 | 8 | hello_world_coroutine = hello_world() 9 | print(hello_world_coroutine) 10 | 11 | event_loop = asyncio.get_event_loop() 12 | try: 13 | print("entering event loop") 14 | result = event_loop.run_until_complete(hello_world_coroutine) 15 | print(result) 16 | finally: 17 | event_loop.close() -------------------------------------------------------------------------------- /Chapter03/06_asyncio-coroutine.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def add_42(number): 5 | print("Adding 42") 6 | return 42 + number 7 | 8 | 9 | async def hello_world(): 10 | print("hello world!") 11 | result = await add_42(23) 12 | return result 13 | 14 | event_loop = asyncio.get_event_loop() 15 | try: 16 | result = event_loop.run_until_complete(hello_world()) 17 | print(result) 18 | finally: 19 | event_loop.close() -------------------------------------------------------------------------------- /Chapter03/08_asyncio-sleep-gather.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def hello_world(): 5 | print("hello world!") 6 | 7 | 8 | async def hello_python(): 9 | print("hello Python!") 10 | await asyncio.sleep(0.1) 11 | 12 | 13 | event_loop = asyncio.get_event_loop() 14 | try: 15 | result = event_loop.run_until_complete(asyncio.gather( 16 | hello_world(), 17 | hello_python(), 18 | )) 19 | print(result) 20 | finally: 21 | event_loop.close() -------------------------------------------------------------------------------- /Chapter03/09_aiohttp-example.py: -------------------------------------------------------------------------------- 1 | import aiohttp 2 | import asyncio 3 | 4 | 5 | async def get(url): 6 | async with aiohttp.ClientSession() as session: 7 | async with session.get(url) as response: 8 | return response 9 | 10 | 11 | loop = asyncio.get_event_loop() 12 | 13 | coroutines = [get("http://example.com") for _ in range(8)] 14 | 15 | results = loop.run_until_complete(asyncio.gather(*coroutines)) 16 | 17 | print("Results: %s" % results) -------------------------------------------------------------------------------- /Chapter03/10_call-later.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | def hello_world(): 5 | print("Hello world!") 6 | 7 | 8 | loop = asyncio.get_event_loop() 9 | loop.call_later(1, hello_world) 10 | loop.run_forever() -------------------------------------------------------------------------------- /Chapter03/11_call-later-repeatedly.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | loop = asyncio.get_event_loop() 5 | 6 | 7 | def hello_world(): 8 | loop.call_later(1, hello_world) 9 | print("Hello world!") 10 | 11 | 12 | loop = asyncio.get_event_loop() 13 | loop.call_later(1, hello_world) 14 | loop.run_forever() -------------------------------------------------------------------------------- /Chapter03/12_asyncio-tcp-server.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | SERVER_ADDRESS = ('127.0.0.1', 1234) 4 | 5 | 6 | class YellEchoServer(asyncio.Protocol): 7 | def connection_made(self, transport): 8 | self.transport = transport 9 | print("Connection received from:", 10 | transport.get_extra_info('peername')) 11 | 12 | def data_received(self, data): 13 | self.transport.write(data.upper()) 14 | 15 | def connection_lost(self, exc): 16 | print("Client disconnected") 17 | 18 | 19 | event_loop = asyncio.get_event_loop() 20 | 21 | factory = event_loop.create_server(YellEchoServer, *SERVER_ADDRESS) 22 | server = event_loop.run_until_complete(factory) 23 | 24 | try: 25 | event_loop.run_forever() 26 | finally: 27 | server.close() 28 | event_loop.run_until_complete(server.wait_closed()) 29 | event_loop.close() -------------------------------------------------------------------------------- /Chapter03/14_asyncio-tcp-client.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | SERVER_ADDRESS = ('127.0.0.1', 1234) 4 | 5 | 6 | class EchoClientProtocol(asyncio.Protocol): 7 | def __init__(self, message, loop): 8 | self.message = message 9 | self.loop = loop 10 | 11 | def talk(self): 12 | self.transport.write(self.message) 13 | 14 | def connection_made(self, transport): 15 | self.transport = transport 16 | self.talk() 17 | 18 | def data_received(self, data): 19 | self.talk() 20 | 21 | def connection_lost(self, exc): 22 | self.loop.stop() 23 | 24 | 25 | loop = asyncio.get_event_loop() 26 | loop.run_until_complete(loop.create_connection( 27 | lambda: EchoClientProtocol(b'Hello World!', loop), 28 | *SERVER_ADDRESS)) 29 | try: 30 | loop.run_forever() 31 | finally: 32 | loop.close() -------------------------------------------------------------------------------- /Chapter03/15_asyncio-server-stats.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import time 3 | 4 | SERVER_ADDRESS = ('127.0.0.1', 1234) 5 | 6 | 7 | class YellEchoServer(asyncio.Protocol): 8 | def __init__(self, stats): 9 | self.stats = stats 10 | self.stats['started at'] = time.time() 11 | 12 | def connection_made(self, transport): 13 | self.transport = transport 14 | self.stats['connections'] += 1 15 | 16 | def data_received(self, data): 17 | self.transport.write(data.upper()) 18 | self.stats['messages sent'] += 1 19 | 20 | 21 | event_loop = asyncio.get_event_loop() 22 | 23 | stats = { 24 | "started at": time.time(), 25 | "connections": 0, 26 | "messages sent": 0, 27 | } 28 | 29 | factory = event_loop.create_server( 30 | lambda: YellEchoServer(stats), *SERVER_ADDRESS) 31 | server = event_loop.run_until_complete(factory) 32 | 33 | try: 34 | event_loop.run_forever() 35 | except KeyboardInterrupt: 36 | pass 37 | finally: 38 | server.close() 39 | event_loop.run_until_complete(server.wait_closed()) 40 | event_loop.close() 41 | 42 | ran_for = time.time() - stats['started at'] 43 | print("Server ran for: %.2f seconds" % ran_for) 44 | print("Connections: %d" % stats['connections']) 45 | print("Messages sent: %d" % stats['messages sent']) 46 | print("Messages sent per second: %.2f" 47 | % (stats['messages sent'] / ran_for)) -------------------------------------------------------------------------------- /Chapter05/01_push-job.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from rq import Queue 4 | from redis import Redis 5 | 6 | q = Queue(connection=Redis()) 7 | 8 | job = q.enqueue(sum, [42, 43]) 9 | # 결과가 준비될 때까지 대기한다. 10 | while job.result is None: 11 | time.sleep(1) 12 | 13 | print(job.result) 14 | -------------------------------------------------------------------------------- /Chapter05/02_push-job-option.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from rq import Queue 4 | from redis import Redis 5 | import requests 6 | 7 | q = Queue(name="http", connection=Redis()) 8 | 9 | job = q.enqueue(requests.get, "http://httpbin.org/delay/1", 10 | ttl=60, result_ttl=300) 11 | # URL을 가져오는 동안, 다른 작업을 할 수 있다. 12 | # 결과가 준비될 때까지 대기한다. 13 | while job.result is None: 14 | time.sleep(1) 15 | 16 | print(job.result) -------------------------------------------------------------------------------- /Chapter05/03_celery-task.py: -------------------------------------------------------------------------------- 1 | import celery 2 | 3 | 4 | app = celery.Celery('03_celery-task', 5 | broker='redis://localhost', 6 | backend='redis://localhost') 7 | 8 | 9 | @app.task 10 | def add(x, y): 11 | return x + y 12 | 13 | 14 | if __name__ == '__main__': 15 | result = add.delay(4, 4) 16 | print("Task state: %s" % result.state) 17 | print("Result: %s" % result.get()) 18 | print("Task state: %s" % result.state) -------------------------------------------------------------------------------- /Chapter05/04_celery-task-retry.py: -------------------------------------------------------------------------------- 1 | import celery 2 | 3 | 4 | app = celery.Celery('04_celery-task-retry', 5 | broker='redis://localhost', 6 | backend='redis://localhost') 7 | 8 | 9 | @app.task(bind=True, retry_backoff=True, 10 | retry_kwargs={'max_retries': 5}) 11 | def add(self, x, y): 12 | try: 13 | return x + y 14 | except OverflowError as exc: 15 | self.retry(exc=exc) 16 | 17 | 18 | if __name__ == '__main__': 19 | result = add.delay(4, 4) 20 | print("Task state: %s" % result.state) 21 | print("Result: %s" % result.get()) 22 | print("Task state: %s" % result.state) -------------------------------------------------------------------------------- /Chapter05/05_celery-chain.py: -------------------------------------------------------------------------------- 1 | import celery 2 | 3 | app = celery.Celery('05_celery-chain', 4 | broker='redis://localhost', 5 | backend='redis://localhost') 6 | 7 | 8 | @app.task 9 | def add(x, y): 10 | return x + y 11 | 12 | 13 | @app.task 14 | def multiply(x, y): 15 | return x * y 16 | 17 | 18 | if __name__ == '__main__': 19 | chain = celery.chain(add.s(4, 6), multiply.s(10)) 20 | print("Chain: %s" % chain) 21 | result = chain() 22 | print("Task state: %s" % result.state) 23 | print("Result: %s" % result.get()) 24 | print("Task state: %s" % result.state) -------------------------------------------------------------------------------- /Chapter05/06_celery-task-queue.py: -------------------------------------------------------------------------------- 1 | import celery 2 | 3 | 4 | app = celery.Celery('06_celery-task-queue', 5 | broker='redis://localhost', 6 | backend='redis://localhost') 7 | 8 | 9 | @app.task 10 | def add(x, y): 11 | return x + y 12 | 13 | 14 | if __name__ == '__main__': 15 | result = add.apply_async(args=[4, 6], queue='low-priority') 16 | print("Task state: %s" % result.state) 17 | print("Result: %s" % result.get()) 18 | print("Task state: %s" % result.state) -------------------------------------------------------------------------------- /Chapter06/01_retrying.py: -------------------------------------------------------------------------------- 1 | while True: 2 | try: 3 | do_something() 4 | except: 5 | pass 6 | else: 7 | break -------------------------------------------------------------------------------- /Chapter06/02_retrying-sleep.py: -------------------------------------------------------------------------------- 1 | import time 2 | import random 3 | 4 | 5 | def do_something(): 6 | if random.randint(0, 1) == 0: 7 | print("Failure") 8 | raise RuntimeError 9 | print("Success") 10 | 11 | 12 | while True: 13 | try: 14 | do_something() 15 | except: 16 | # 재시도 전에 1초간 sleep 17 | time.sleep(1) 18 | else: 19 | break -------------------------------------------------------------------------------- /Chapter06/03_retrying-backoff.py: -------------------------------------------------------------------------------- 1 | import time 2 | import random 3 | 4 | 5 | def do_something(): 6 | if random.randint(0, 1) == 0: 7 | print("Failure") 8 | raise RuntimeError 9 | print("Success") 10 | 11 | 12 | attempt = 0 13 | while True: 14 | try: 15 | do_something() 16 | except: 17 | # 재시도 전에 2^attempt 동안 sleep 18 | time.sleep(2 ** attempt) 19 | attempt += 1 20 | else: 21 | break -------------------------------------------------------------------------------- /Chapter06/04_retrying-tenacity.py: -------------------------------------------------------------------------------- 1 | import tenacity 2 | import random 3 | 4 | 5 | def do_something(): 6 | if random.randint(0, 1) == 0: 7 | print("Failure") 8 | raise RuntimeError 9 | print("Success") 10 | 11 | 12 | tenacity.Retrying()(do_something) -------------------------------------------------------------------------------- /Chapter06/05_tenacity-fixed-waiting.py: -------------------------------------------------------------------------------- 1 | import tenacity 2 | import random 3 | 4 | 5 | def do_something(): 6 | if random.randint(0, 1) == 0: 7 | print("Failure") 8 | raise RuntimeError 9 | print("Success") 10 | 11 | 12 | @tenacity.retry(wait=tenacity.wait_fixed(1)) 13 | def do_something_and_retry(): 14 | do_something() 15 | 16 | 17 | do_something_and_retry() -------------------------------------------------------------------------------- /Chapter06/06_tenacity-backoff.py: -------------------------------------------------------------------------------- 1 | import tenacity 2 | import random 3 | 4 | 5 | def do_something(): 6 | if random.randint(0, 1) == 0: 7 | print("Failure") 8 | raise RuntimeError 9 | print("Success") 10 | 11 | 12 | @tenacity.retry( 13 | wait=tenacity.wait_exponential(multiplier=0.5, max=30, exp_base=2), 14 | ) 15 | def do_something_and_retry(): 16 | do_something() 17 | 18 | 19 | do_something_and_retry() -------------------------------------------------------------------------------- /Chapter06/07_tenacity-combine-wait.py: -------------------------------------------------------------------------------- 1 | import tenacity 2 | import random 3 | 4 | 5 | def do_something(): 6 | if random.randint(0, 1) == 0: 7 | print("Failure") 8 | raise RuntimeError 9 | print("Success") 10 | 11 | 12 | @tenacity.retry( 13 | wait=tenacity.wait_fixed(10) + tenacity.wait_random(0, 3) 14 | ) 15 | def do_something_and_retry(): 16 | do_something() 17 | 18 | 19 | do_something_and_retry() -------------------------------------------------------------------------------- /Chapter06/08_tenacity-specific-condition.py: -------------------------------------------------------------------------------- 1 | import tenacity 2 | import random 3 | 4 | 5 | def do_something(): 6 | if random.randint(0, 1) == 0: 7 | print("Failure") 8 | raise RuntimeError 9 | print("Success") 10 | 11 | 12 | @tenacity.retry(wait=tenacity.wait_fixed(1), 13 | retry=tenacity.retry_if_exception_type(RuntimeError)) 14 | def do_something_and_retry(): 15 | return do_something() 16 | 17 | 18 | do_something_and_retry() -------------------------------------------------------------------------------- /Chapter06/09_tenacity-combine-condition.py: -------------------------------------------------------------------------------- 1 | import tenacity 2 | import random 3 | 4 | 5 | def do_something(): 6 | if random.randint(0, 1) == 0: 7 | print("Failure") 8 | raise RuntimeError 9 | print("Success") 10 | return True 11 | 12 | 13 | @tenacity.retry(wait=tenacity.wait_fixed(1), 14 | stop=tenacity.stop_after_delay(60), 15 | retry=(tenacity.retry_if_exception_type(RuntimeError) | 16 | tenacity.retry_if_result( 17 | lambda result: result is None))) 18 | def do_something_and_retry(): 19 | return do_something() 20 | 21 | 22 | do_something_and_retry() -------------------------------------------------------------------------------- /Chapter06/10_tenacity-without-decorator.py: -------------------------------------------------------------------------------- 1 | import tenacity 2 | import random 3 | 4 | def do_something(): 5 | if random.randint(0, 1) == 0: 6 | print("Failure") 7 | raise IOError 8 | print("Success") 9 | return True 10 | 11 | 12 | r = tenacity.Retrying( 13 | wait=tenacity.wait_fixed(1), 14 | retry=tenacity.retry_if_exception_type(IOError)) 15 | r.call(do_something) -------------------------------------------------------------------------------- /Chapter07/01_threading-lock.py: -------------------------------------------------------------------------------- 1 | import threading 2 | 3 | stdout_lock = threading.Lock() 4 | 5 | 6 | def print_something(something): 7 | with stdout_lock: 8 | print(something) 9 | 10 | 11 | t = threading.Thread(target=print_something, args=("hello",)) 12 | t.daemon = True 13 | t.start() 14 | print_something("thread started") -------------------------------------------------------------------------------- /Chapter07/02_threading-reentrant-lock.py: -------------------------------------------------------------------------------- 1 | import threading 2 | 3 | rlock = threading.RLock() 4 | 5 | with rlock: 6 | with rlock: 7 | print("Double acquired") -------------------------------------------------------------------------------- /Chapter07/03_threading-event.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import time 3 | 4 | 5 | stop = threading.Event() 6 | 7 | 8 | def background_job(): 9 | while not stop.is_set(): 10 | print("I'm still running!") 11 | stop.wait(0.1) 12 | 13 | 14 | t = threading.Thread(target=background_job) 15 | t.start() 16 | print("thread started") 17 | time.sleep(1) 18 | stop.set() 19 | t.join() -------------------------------------------------------------------------------- /Chapter07/04_multiprocessing-without-lock.py: -------------------------------------------------------------------------------- 1 | import multiprocessing 2 | import time 3 | 4 | 5 | def print_cat(): 6 | # 조금 기다려서 임의의 결과가 나오도록 한다. 7 | time.sleep(0.1) 8 | print(" /\\_/\\") 9 | print("( o.o )") 10 | print(" > ^ <") 11 | 12 | 13 | with multiprocessing.Pool(processes=3) as pool: 14 | jobs = [] 15 | for _ in range(5): 16 | jobs.append(pool.apply_async(print_cat)) 17 | for job in jobs: 18 | job.wait() -------------------------------------------------------------------------------- /Chapter07/05_multiprocessing-lock.py: -------------------------------------------------------------------------------- 1 | import multiprocessing 2 | import time 3 | 4 | 5 | stdout_lock = multiprocessing.Lock() 6 | 7 | 8 | def print_cat(): 9 | # 조금 기다려서 임의의 결과가 나오도록 한다. 10 | time.sleep(0.1) 11 | with stdout_lock: 12 | print(" /\\_/\\") 13 | print("( o.o )") 14 | print(" > ^ <") 15 | 16 | 17 | with multiprocessing.Pool(processes=3) as pool: 18 | jobs = [] 19 | for _ in range(5): 20 | jobs.append(pool.apply_async(print_cat)) 21 | for job in jobs: 22 | job.wait() -------------------------------------------------------------------------------- /Chapter07/06_fasteners-interprocess-lock.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import fasteners 4 | 5 | 6 | lock = fasteners.InterProcessLock("/tmp/mylock") 7 | with lock: 8 | print("Access locked") 9 | time.sleep(1) -------------------------------------------------------------------------------- /Chapter07/07_fasteners-decorator.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import fasteners 4 | 5 | 6 | @fasteners.interprocess_locked('/tmp/tmp_lock_file') 7 | def locked_print(): 8 | for i in range(10): 9 | print('I have the lock') 10 | time.sleep(0.1) 11 | 12 | 13 | locked_print() -------------------------------------------------------------------------------- /Chapter07/08_etcd-lock.py: -------------------------------------------------------------------------------- 1 | import etcd3 2 | 3 | client = etcd3.client() 4 | lock = client.lock("foobar") 5 | lock.acquire() 6 | try: 7 | print("do something") 8 | finally: 9 | lock.release() -------------------------------------------------------------------------------- /Chapter07/09_etcd-lock-with.py: -------------------------------------------------------------------------------- 1 | import etcd3 2 | 3 | client = etcd3.client() 4 | lock = client.lock("foobar") 5 | with lock: 6 | print("do something") -------------------------------------------------------------------------------- /Chapter07/10_etcd-cotyledon.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import time 3 | 4 | import cotyledon 5 | import etcd3 6 | 7 | 8 | class PrinterService(cotyledon.Service): 9 | name = "printer" 10 | 11 | def __init__(self, worker_id): 12 | super(PrinterService, self).__init__(worker_id) 13 | self._shutdown = threading.Event() 14 | self.client = etcd3.client() 15 | 16 | def run(self): 17 | while not self._shutdown.is_set(): 18 | with self.client.lock("print"): 19 | print("I'm %s and I'm the only one printing" 20 | % self.worker_id) 21 | time.sleep(1) 22 | 23 | def terminate(self): 24 | self._shutdown.set() 25 | 26 | 27 | # 매니저 생성 28 | manager = cotyledon.ServiceManager() 29 | # 실행할 4개의 PrinterService 추가 30 | manager.add(PrinterService, 4) 31 | # 모두 실행 32 | manager.run() -------------------------------------------------------------------------------- /Chapter07/11_tooz-coodinator.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | from tooz import coordination 4 | 5 | # UUID를 사용해서 고유 식별자 생성 6 | identifier = str(uuid.uuid4()) 7 | # Coordinator 객체 얻음 8 | c = coordination.get_coordinator( 9 | "etcd3://localhost", identifier) 10 | # Coordinator 시작(연결 시작) 11 | c.start(start_heart=True) 12 | # 프로그램이 끝나면 Coordinator도 종료 13 | c.stop() -------------------------------------------------------------------------------- /Chapter07/12_tooz-coodinator-lock.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | from tooz import coordination 4 | 5 | # UUID를 사용해서 고유 식별자 생성 6 | identifier = str(uuid.uuid4()) 7 | # Coordinator 객체 얻음 8 | c = coordination.get_coordinator( 9 | "etcd3://localhost", identifier) 10 | # Coordinator 시작(연결 시작) 11 | c.start(start_heart=True) 12 | 13 | lock = c.get_lock(b"name_of_the_lock") 14 | # 락 획득 및 해제 15 | assert lock.acquire() is True 16 | assert lock.release() is True 17 | 18 | # 획득하지 않은 락은 해제 못함 19 | assert lock.release() is False 20 | 21 | assert lock.acquire() is True 22 | #블록 하지 않고 락 획득 시도(실패해도 바로 알 수 있음.) 23 | assert lock.acquire(blocking=False) is False 24 | # 락 획득 시도 시에 5초간 대기 25 | assert lock.acquire(blocking=5) is False 26 | assert lock.release() is True 27 | 28 | # 프로그램이 끝나면 Coordinator도 종료 29 | c.stop() -------------------------------------------------------------------------------- /Chapter07/13_tooz-with.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | from tooz import coordination 4 | 5 | # UUID를 사용해서 고유 식별자 생성 6 | identifier = str(uuid.uuid4()) 7 | # Coordinator 객체 얻음 8 | c = coordination.get_coordinator( 9 | "etcd3://localhost", identifier) 10 | # Coordinator 시작(연결 시작) 11 | c.start(start_heart=True) 12 | 13 | lock = c.get_lock(b"foobar") 14 | with lock: 15 | print("do something") -------------------------------------------------------------------------------- /Chapter08/01_join-group.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import time 3 | 4 | from tooz import coordination 5 | 6 | # client id와 group id가 인수로 전달됐는지 확인. 7 | if len(sys.argv) != 3: 8 | print("Usage: %s " % sys.argv[0]) 9 | sys.exit(1) 10 | 11 | # Coordinator 객체 얻음 12 | c = coordination.get_coordinator( 13 | "etcd3://localhost", 14 | sys.argv[1].encode()) 15 | # Coordinator 시작(연결 시작) 16 | c.start(start_heart=True) 17 | 18 | group = sys.argv[2].encode() 19 | 20 | # 그룹 생성 21 | try: 22 | c.create_group(group).get() 23 | except coordination.GroupAlreadyExist: 24 | pass 25 | 26 | # 그룹 참가 27 | c.join_group(group).get() 28 | 29 | # 멤버 목록 출력 30 | members = c.get_members(group) 31 | print(members.get()) 32 | 33 | # 5초간 대기 34 | time.sleep(5) 35 | 36 | # 그룹 나가기 37 | c.leave_group(group).get() 38 | 39 | # 프로그램이 끝나면 Coordinator도 종료 40 | c.stop() -------------------------------------------------------------------------------- /Chapter08/02_join-group-capabilities.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import time 3 | 4 | from tooz import coordination 5 | 6 | # client id, group id 및 mood가 인수로 전달됐는지 확인. 7 | if len(sys.argv) != 4: 8 | print("Usage: %s " 9 | % sys.argv[0]) 10 | sys.exit(1) 11 | 12 | # Coordinator 객체 얻음 13 | c = coordination.get_coordinator( 14 | "etcd3://localhost", 15 | sys.argv[1].encode()) 16 | # Coordinator 시작(연결 시작) 17 | c.start(start_heart=True) 18 | 19 | group = sys.argv[2].encode() 20 | 21 | # 그룹 생성 22 | try: 23 | c.create_group(group).get() 24 | except coordination.GroupAlreadyExist: 25 | pass 26 | 27 | # 그룹 참가 28 | c.join_group( 29 | group, 30 | capabilities={"mood": sys.argv[3]} 31 | ).get() 32 | 33 | # 멤버 목록과 역량을 출력한다. 34 | # API가 비동기 방식이므로, 역량을 얻기 위한 요청을 35 | # 동시에 보내서 병렬로 실행되도록 한다. 36 | get_capabilities = [ 37 | (member, c.get_member_capabilities(group, member)) 38 | for member in c.get_members(group).get() 39 | ] 40 | 41 | for member, cap in get_capabilities: 42 | print("Member %s has capabilities: %s" 43 | % (member, cap.get())) 44 | 45 | # 5초간 대기 46 | time.sleep(5) 47 | 48 | # 그룹 나가기 49 | c.leave_group(group).get() 50 | 51 | # 프로그램이 끝나면 Coordinator도 종료 52 | c.stop() 53 | -------------------------------------------------------------------------------- /Chapter08/03_join-watchers.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import time 3 | 4 | from tooz import coordination 5 | 6 | # client id와 group id가 인수로 전달됐는지 확인. 7 | if len(sys.argv) != 3: 8 | print("Usage: %s " % sys.argv[0]) 9 | sys.exit(1) 10 | 11 | # Coordinator 객체 얻음 12 | c = coordination.get_coordinator( 13 | #효과를 빨리 확인하기 위해 타임아웃을 짧게 설정함. 14 | "etcd3://localhost/?timeout=3", 15 | sys.argv[1].encode()) 16 | # Coordinator 시작(연결 시작) 17 | c.start(start_heart=True) 18 | 19 | group = sys.argv[2].encode() 20 | 21 | # 그룹 생성 22 | try: 23 | c.create_group(group).get() 24 | except coordination.GroupAlreadyExist: 25 | pass 26 | 27 | # 그룹 참가 28 | c.join_group(group).get() 29 | 30 | # 그룹에 참가하거나 나갈 때 호출되도록 31 | # print 함수를 등록한다. 32 | c.watch_join_group(group, print) 33 | c.watch_leave_group(group, print) 34 | 35 | while True: 36 | c.run_watchers() 37 | time.sleep(2) 38 | 39 | # 그룹 나가기 40 | c.leave_group(group).get() 41 | 42 | # 프로그램이 끝나면 Coordinator도 종료 43 | c.stop() -------------------------------------------------------------------------------- /Chapter08/05_tooz-hash-ring.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | from tooz import hashring 3 | 4 | 5 | NUMBER_OF_NODES = 16 6 | 7 | # Step #1 16개의 노드로 해시 링 생성 8 | hr = hashring.HashRing(["node%d" % i for i in range(NUMBER_OF_NODES)]) 9 | nodes = hr.get_nodes(b"some data") 10 | print(nodes) 11 | nodes = hr.get_nodes(b"some data", replicas=2) 12 | print(nodes) 13 | nodes = hr.get_nodes(b"some other data", replicas=3) 14 | print(nodes) 15 | nodes = hr.get_nodes(b"some other of my data", replicas=2) 16 | print(nodes) 17 | 18 | # Step #2 노드 하나를 제거 19 | print("Removing node8") 20 | hr.remove_node("node8") 21 | nodes = hr.get_nodes(b"some data") 22 | print(nodes) 23 | nodes = hr.get_nodes(b"some data", replicas=2) 24 | print(nodes) 25 | nodes = hr.get_nodes(b"some other data", replicas=3) 26 | print(nodes) 27 | nodes = hr.get_nodes(b"some other of my data", replicas=2) 28 | print(nodes) 29 | 30 | # Step #3 새 노드 추가 31 | print("Adding node17") 32 | hr.add_node("node17") 33 | nodes = hr.get_nodes(b"some data") 34 | print(nodes) 35 | nodes = hr.get_nodes(b"some data", replicas=2) 36 | print(nodes) 37 | nodes = hr.get_nodes(b"some other data", replicas=3) 38 | print(nodes) 39 | nodes = hr.get_nodes(b"some other of my data", replicas=2) 40 | print(nodes) 41 | nodes = hr.get_nodes(b"some data that should end on node17", replicas=2) 42 | print(nodes) 43 | 44 | # Step #4 제거한 노드를, 더 높은 가중치로 다시 추가한다. 45 | print("Adding back node8 with weight") 46 | hr.add_node("node8", weight=100) 47 | nodes = hr.get_nodes(b"some data") 48 | print(nodes) 49 | nodes = hr.get_nodes(b"some data", replicas=2) 50 | print(nodes) 51 | nodes = hr.get_nodes(b"some other data", replicas=3) 52 | print(nodes) 53 | nodes = hr.get_nodes(b"some other of my data", replicas=2) 54 | print(nodes) -------------------------------------------------------------------------------- /Chapter08/06_tooz-join-partitioned-group.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import time 3 | 4 | from tooz import coordination 5 | 6 | # client id와 group id가 인수로 전달됐는지 확인. 7 | if len(sys.argv) != 3: 8 | print("Usage: %s " % sys.argv[0]) 9 | sys.exit(1) 10 | 11 | # Coordinator 객체 얻음 12 | c = coordination.get_coordinator( 13 | "etcd3://localhost", 14 | sys.argv[1].encode()) 15 | # Coordinator 시작(연결 시작) 16 | c.start(start_heart=True) 17 | 18 | group = sys.argv[2].encode() 19 | 20 | # 파티션 그룹에 참가 21 | p = c.join_partitioned_group(group) 22 | 23 | print(p.members_for_object("foobar")) 24 | 25 | # 5초간 대기 26 | time.sleep(5) 27 | 28 | # 그룹 나가기 29 | c.leave_group(group).get() 30 | 31 | # 프로그램이 끝나면 Coordinator도 종료 32 | c.stop() -------------------------------------------------------------------------------- /Chapter08/07_spread-fetching-web.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import uuid 3 | 4 | import requests 5 | from tooz import coordination 6 | 7 | 8 | class URL(str): 9 | def __tooz_hash__(self): 10 | # URL을 고유 식별자로 사용한다. 11 | return self.encode() 12 | 13 | urls_to_fetch = [ 14 | URL("https://httpbin.org/bytes/%d" % n) 15 | for n in range(100) 16 | ] 17 | 18 | GROUP_NAME = b"fetcher" 19 | MEMBER_ID = str(uuid.uuid4()).encode('ascii') 20 | 21 | # Coordinator 객체 얻음 22 | c = coordination.get_coordinator("etcd3://localhost", MEMBER_ID) 23 | # Coordinator 시작(연결 시작) 24 | c.start(start_heart=True) 25 | 26 | # 파티션 그룹에 참가 27 | p = c.join_partitioned_group(GROUP_NAME) 28 | 29 | try: 30 | for url in itertools.cycle(urls_to_fetch): 31 | # 멤버십 변경이 없는지 확인 32 | c.run_watchers() 33 | # print("%s -> %s" % (url, p.members_for_object(url))) 34 | if p.belongs_to_self(url): 35 | try: 36 | r = requests.get(url) 37 | except Exception: 38 | # 에러가 발생하면 39 | # 단순히 다음 아이템으로 이동 40 | pass 41 | else: 42 | print("%s: fetched %s (%d)" 43 | % (MEMBER_ID, r.url, r.status_code)) 44 | finally: 45 | # 그룹 나가기 46 | c.leave_group(GROUP_NAME).get() -------------------------------------------------------------------------------- /Chapter09/01_WSGI-basic.py: -------------------------------------------------------------------------------- 1 | def application(environ, start_response): 2 | """가장 간단한 애플리케이션 객체""" 3 | status = '200 OK' 4 | response_headers = [('Content-type', 'text/plain')] 5 | start_response(status, response_headers) 6 | return [b'Hello world!\n'] 7 | -------------------------------------------------------------------------------- /Chapter09/02_WSGI-application.py: -------------------------------------------------------------------------------- 1 | from wsgiref.simple_server import make_server 2 | 3 | 4 | def application(environ, start_response): 5 | """text/plain으로 wsgi 환경변수를 반환.""" 6 | body = '\n'.join([ 7 | '%s: %s' % (key, value) for key, value in sorted(environ.items()) 8 | ]) 9 | 10 | start_response("200 OK", [ 11 | ('Content-Type', 'text/plain'), 12 | ('Content-Length', str(len(body))) 13 | ]) 14 | 15 | return [body.encode()] 16 | 17 | 18 | # 서버 초기화 19 | httpd = make_server('localhost', 8051, application) 20 | # 요청 하나를 기다려서 처리한 뒤에 종료 21 | httpd.handle_request() 22 | # 요청 및 응답을 확인하려면 'curl -v http://localhost:8051' 명령 실행. -------------------------------------------------------------------------------- /Chapter09/04_publish-command.py: -------------------------------------------------------------------------------- 1 | import redis 2 | 3 | r = redis.Redis() 4 | r.publish("chatroom", "hello world") -------------------------------------------------------------------------------- /Chapter09/06_message-listen.py: -------------------------------------------------------------------------------- 1 | import redis 2 | 3 | r = redis.Redis() 4 | p = r.pubsub() 5 | p.subscribe("chatroom") 6 | for message in p.listen(): 7 | print(message) -------------------------------------------------------------------------------- /Chapter09/08_flask-streamer.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import flask 4 | import redis 5 | 6 | 7 | application = flask.Flask(__name__) 8 | 9 | 10 | def stream_messages(channel): 11 | r = redis.Redis() 12 | p = r.pubsub() 13 | p.subscribe(channel) 14 | for message in p.listen(): 15 | if message["type"] == "message": 16 | yield "data: " + json.dumps(message["data"].decode()) + "\n\n" 17 | 18 | @application.route("/message/", methods=['GET']) 19 | def get_messages(channel): 20 | return flask.Response( 21 | flask.stream_with_context(stream_messages(channel)), 22 | mimetype='text/event-stream') 23 | 24 | 25 | @application.route("/message/", methods=['POST']) 26 | def send_message(channel): 27 | data = flask.request.json 28 | if (not data or 'source' not in data or 'content' not in data): 29 | flask.abort(400) 30 | r = redis.Redis() 31 | r.publish(channel, "<{}> {}".format(data["source"], data["content"])) 32 | return "", 202 33 | -------------------------------------------------------------------------------- /Chapter09/11_flask-etag.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import flask 3 | import werkzeug 4 | 5 | 6 | application = flask.Flask(__name__) 7 | 8 | 9 | class NotModified(werkzeug.exceptions.HTTPException): 10 | code = 304 11 | 12 | 13 | @application.route("/", methods=['GET']) 14 | def get_index(): 15 | # 이 예제는 항상 동일한 컨텐츠를 사용하므로, Etag도 고정된 값이다. 16 | ETAG = "hword" 17 | 18 | if_match = flask.request.headers.get("If-Match") 19 | if if_match is not None and if_match != ETAG: 20 | raise NotModified 21 | 22 | if_none_match = flask.request.headers.get("If-None-Match") 23 | if if_none_match is not None and if_none_match == ETAG: 24 | raise NotModified 25 | 26 | return flask.Response("hello world", headers={"ETag": "hword"}) 27 | 28 | 29 | class TestApp(unittest.TestCase): 30 | def test_get_index(self): 31 | test_app = application.test_client() 32 | result = test_app.get() 33 | self.assertEqual(200, result.status_code) 34 | 35 | def test_get_index_if_match_positive(self): 36 | test_app = application.test_client() 37 | result = test_app.get(headers={"If-Match": "hword"}) 38 | self.assertEqual(200, result.status_code) 39 | 40 | def test_get_index_if_match_negative(self): 41 | test_app = application.test_client() 42 | result = test_app.get(headers={"If-Match": "foobar"}) 43 | self.assertEqual(304, result.status_code) 44 | 45 | def test_get_index_if_none_match_positive(self): 46 | test_app = application.test_client() 47 | result = test_app.get(headers={"If-None-Match": "hword"}) 48 | self.assertEqual(304, result.status_code) 49 | 50 | def test_get_index_if_none_match_negative(self): 51 | test_app = application.test_client() 52 | result = test_app.get(headers={"If-None-Match": "foobar"}) 53 | self.assertEqual(200, result.status_code) 54 | 55 | 56 | if __name__ == "__main__": 57 | application.run() 58 | -------------------------------------------------------------------------------- /Chapter09/12_flask-etag-put.py: -------------------------------------------------------------------------------- 1 | import random 2 | import unittest 3 | 4 | import flask 5 | from werkzeug import exceptions 6 | 7 | 8 | app = flask.Flask(__name__) 9 | 10 | 11 | class NotModified(exceptions.HTTPException): 12 | code = 304 13 | 14 | 15 | ETAG = random.randint(1000, 5000) 16 | VALUE = "hello" 17 | 18 | 19 | def check_etag(exception_class): 20 | global ETAG 21 | 22 | if_match = flask.request.headers.get("If-Match") 23 | if if_match is not None and if_match != str(ETAG): 24 | raise exception_class 25 | 26 | if_none_match = flask.request.headers.get("If-None-Match") 27 | if if_none_match is not None and if_none_match == str(ETAG): 28 | raise exception_class 29 | 30 | 31 | @app.route("/", methods=['GET']) 32 | def get_index(): 33 | check_etag(NotModified) 34 | return flask.Response(VALUE, headers={"ETag": ETAG}) 35 | 36 | 37 | @app.route("/", methods=['PUT']) 38 | def put_index(): 39 | global ETAG, VALUE 40 | 41 | check_etag(exceptions.PreconditionFailed) 42 | 43 | ETAG += random.randint(3, 9) 44 | VALUE = flask.request.data 45 | return flask.Response(VALUE, headers={"ETag": ETAG}) 46 | 47 | 48 | class TestApp(unittest.TestCase): 49 | def test_put_index_if_match_positive(self): 50 | test_app = app.test_client() 51 | resp = test_app.get() 52 | etag = resp.headers["ETag"] 53 | new_value = b"foobar" 54 | result = test_app.put(headers={"If-Match": etag}, 55 | data=new_value) 56 | self.assertEqual(200, result.status_code) 57 | self.assertEqual(new_value, result.data) 58 | 59 | def test_put_index_if_match_negative(self): 60 | test_app = app.test_client() 61 | result = test_app.put(headers={"If-Match": "wrong"}) 62 | self.assertEqual(412, result.status_code) 63 | 64 | 65 | if __name__ == "__main__": 66 | app.run() -------------------------------------------------------------------------------- /Chapter09/13_flask-etag-retry.py: -------------------------------------------------------------------------------- 1 | import random 2 | import unittest 3 | 4 | import flask 5 | from werkzeug import exceptions 6 | 7 | 8 | app = flask.Flask(__name__) 9 | 10 | 11 | class NotModified(exceptions.HTTPException): 12 | code = 304 13 | 14 | 15 | ETAG = random.randint(1000, 5000) 16 | VALUE = "hello" 17 | 18 | 19 | def check_etag(exception_class): 20 | global ETAG 21 | 22 | if_match = flask.request.headers.get("If-Match") 23 | if if_match is not None and if_match != str(ETAG): 24 | raise exception_class 25 | 26 | if_none_match = flask.request.headers.get("If-None-Match") 27 | if if_none_match is not None and if_none_match == str(ETAG): 28 | raise exception_class 29 | 30 | 31 | @app.route("/", methods=['GET']) 32 | def get_index(): 33 | check_etag(NotModified) 34 | return flask.Response(VALUE, headers={"ETag": ETAG}) 35 | 36 | 37 | @app.route("/", methods=['PUT']) 38 | def put_index(): 39 | global ETAG, VALUE 40 | 41 | check_etag(exceptions.PreconditionFailed) 42 | 43 | ETAG += random.randint(3, 9) 44 | VALUE = flask.request.data 45 | return flask.Response(VALUE, headers={"ETag": ETAG}) 46 | 47 | 48 | class TestApp(unittest.TestCase): 49 | def test_put_index_if_match_positive(self): 50 | test_app = app.test_client() 51 | resp = test_app.get() 52 | etag = resp.headers["ETag"] 53 | new_value = b"foobar" 54 | result = test_app.put(headers={"If-Match": etag}, 55 | data=new_value) 56 | self.assertEqual(200, result.status_code) 57 | self.assertEqual(new_value, result.data) 58 | 59 | def test_put_index_if_match_negative(self): 60 | test_app = app.test_client() 61 | result = test_app.put(headers={"If-Match": "wrong"}) 62 | self.assertEqual(412, result.status_code) 63 | 64 | while True: 65 | resp = test_app.get() 66 | etag = resp.headers["ETag"] 67 | 68 | #필요한 작업을 처리한다. 69 | new_data = b"boobar" 70 | 71 | resp = test_app.put(data=new_data, headers={"If-Match": etag}) 72 | if resp.status_code == 200: 73 | break 74 | elif resp.status_code == 412: 75 | continue 76 | else: 77 | raise RuntimeError("Unknown exception: %d" % resp.status_code) 78 | 79 | self.assertEqual(200, resp.status_code) 80 | self.assertEqual(new_data, result.data) 81 | 82 | 83 | if __name__ == "__main__": 84 | app.run() -------------------------------------------------------------------------------- /Chapter09/14_flask-async-job.py: -------------------------------------------------------------------------------- 1 | import queue 2 | import threading 3 | import uuid 4 | 5 | import flask 6 | from werkzeug import routing 7 | 8 | 9 | application = flask.Flask(__name__) 10 | JOBS = queue.Queue() 11 | RESULTS = {} 12 | 13 | 14 | class UUIDConverter(routing.BaseConverter): 15 | 16 | @staticmethod 17 | def to_python(value): 18 | try: 19 | return uuid.UUID(value) 20 | except ValueError: 21 | raise routing.ValidationError 22 | 23 | @staticmethod 24 | def to_url(value): 25 | return str(value) 26 | 27 | 28 | application.url_map.converters['uuid'] = UUIDConverter 29 | 30 | 31 | @application.route("/sum/", methods=['GET']) 32 | def get_job(job): 33 | if job not in RESULTS: 34 | return flask.Response(status=404) 35 | if RESULTS[job] is None: 36 | return flask.jsonify({"status": "waiting"}) 37 | return flask.jsonify({"status": "done", "result": RESULTS[job]}) 38 | 39 | 40 | @application.route("/sum", methods=['POST']) 41 | def post_job(): 42 | # 무작위로 작업 id를 생성한다. 43 | job_id = uuid.uuid4() 44 | # 실행할 작업을 저장한다. 45 | RESULTS[job_id] = None 46 | JOBS.put((job_id, flask.request.args.getlist('number', type=float) 47 | )) 48 | return flask.Response( 49 | headers={"Location": flask.url_for("get_job", job=job_id)}, 50 | status=202) 51 | 52 | 53 | def compute_jobs(): 54 | while True: 55 | job_id, number = JOBS.get() 56 | RESULTS[job_id] = sum(number) 57 | 58 | if __name__ == "__main__": 59 | t = threading.Thread(target=compute_jobs) 60 | t.daemon = True 61 | t.start() 62 | application.run(debug=True) -------------------------------------------------------------------------------- /Chapter09/15_requests-session.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | session = requests.Session() 4 | session.get("http://example.com") 5 | # 이미 맺어진 연결을 재사용 6 | session.get("http://example.com") -------------------------------------------------------------------------------- /Chapter09/16_requests-connection-pool.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | 4 | session = requests.Session() 5 | adapter = requests.adapters.HTTPAdapter( 6 | pool_connections=100, 7 | pool_maxsize=100) 8 | session.mount('http://', adapter) 9 | response = session.get("http://example.org") -------------------------------------------------------------------------------- /Chapter09/17_requests-futures.py: -------------------------------------------------------------------------------- 1 | from concurrent import futures 2 | 3 | import requests 4 | 5 | 6 | with futures.ThreadPoolExecutor(max_workers=4) as executor: 7 | futures = [ 8 | executor.submit( 9 | lambda: requests.get("http://example.org")) 10 | for _ in range(8) 11 | ] 12 | 13 | results = [ 14 | f.result().status_code 15 | for f in futures 16 | ] 17 | 18 | print("Results: %s" % results) -------------------------------------------------------------------------------- /Chapter09/18_requests-futures_exam.py: -------------------------------------------------------------------------------- 1 | from requests_futures import sessions 2 | 3 | 4 | session = sessions.FuturesSession() 5 | 6 | futures = [ 7 | session.get("http://example.org") 8 | for _ in range(8) 9 | ] 10 | 11 | results = [ 12 | f.result().status_code 13 | for f in futures 14 | ] 15 | 16 | print("Results: %s" % results) -------------------------------------------------------------------------------- /Chapter09/19_aiohttp_exam.py: -------------------------------------------------------------------------------- 1 | import aiohttp 2 | import asyncio 3 | 4 | 5 | async def get(url): 6 | async with aiohttp.ClientSession() as session: 7 | async with session.get(url) as response: 8 | return response 9 | 10 | 11 | loop = asyncio.get_event_loop() 12 | 13 | coroutines = [get("http://example.com") for _ in range(8)] 14 | 15 | results = loop.run_until_complete(asyncio.gather(*coroutines)) 16 | 17 | print("Results: %s" % results) -------------------------------------------------------------------------------- /Chapter09/20_requests-comparison.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import time 3 | 4 | import aiohttp 5 | import asyncio 6 | import requests 7 | from requests_futures import sessions 8 | 9 | URL = "http://httpbin.org/delay/1" 10 | TRIES = 10 11 | 12 | 13 | @contextlib.contextmanager 14 | def report_time(test): 15 | t0 = time.time() 16 | yield 17 | print("Time needed for `%s' called: %.2fs" 18 | % (test, time.time() - t0)) 19 | 20 | 21 | with report_time("serialized"): 22 | for i in range(TRIES): 23 | requests.get(URL) 24 | 25 | 26 | session = requests.Session() 27 | with report_time("Session"): 28 | for i in range(TRIES): 29 | session.get(URL) 30 | 31 | 32 | session = sessions.FuturesSession(max_workers=2) 33 | with report_time("FuturesSession w/ 2 workers"): 34 | futures = [session.get(URL) 35 | for i in range(TRIES)] 36 | for f in futures: 37 | f.result() 38 | 39 | 40 | session = sessions.FuturesSession(max_workers=TRIES) 41 | with report_time("FuturesSession w/ max workers"): 42 | futures = [session.get(URL) 43 | for i in range(TRIES)] 44 | for f in futures: 45 | f.result() 46 | 47 | 48 | async def get(url): 49 | async with aiohttp.ClientSession() as session: 50 | async with session.get(url) as response: 51 | await response.read() 52 | 53 | loop = asyncio.get_event_loop() 54 | with report_time("aiohttp"): 55 | loop.run_until_complete( 56 | asyncio.gather(*[get(URL) 57 | for i in range(TRIES)])) -------------------------------------------------------------------------------- /Chapter09/22_requests-streaming.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | 4 | # 'with'를 사용해서 응답 stream을 확실히 닫아서 5 | # 연결을 다시 pool에 반환한다. 6 | with requests.get('http://example.org', stream=True) as r: 7 | print(list(r.iter_content())) -------------------------------------------------------------------------------- /Chapter09/23_aiohttp-streaming.py: -------------------------------------------------------------------------------- 1 | import aiohttp 2 | import asyncio 3 | 4 | 5 | async def get(url): 6 | async with aiohttp.ClientSession() as session: 7 | async with session.get(url) as response: 8 | return await response.content.read() 9 | 10 | loop = asyncio.get_event_loop() 11 | tasks = [asyncio.ensure_future(get("http://example.com"))] 12 | loop.run_until_complete(asyncio.wait(tasks)) 13 | print("Results: %s" % [task.result() for task in tasks]) -------------------------------------------------------------------------------- /Chapter09/25_gabbi-flask.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import flask 4 | from gabbi import driver 5 | import werkzeug 6 | 7 | 8 | application = flask.Flask(__name__) 9 | 10 | 11 | class NotModified(werkzeug.exceptions.HTTPException): 12 | code = 304 13 | 14 | 15 | @application.route("/", methods=['GET']) 16 | def get_index(): 17 | # 이 예제는 항상 동일한 컨텐츠를 사용하므로, Etag도 고정된 값이다. 18 | ETAG = "hword" 19 | 20 | if_match = flask.request.headers.get("If-Match") 21 | if if_match is not None and if_match != ETAG: 22 | raise NotModified 23 | 24 | if_none_match = flask.request.headers.get("If-None-Match") 25 | if if_none_match is not None and if_none_match == ETAG: 26 | raise NotModified 27 | 28 | return flask.Response("hello world", 29 | headers={"ETag": "hword"}) 30 | 31 | # 아래 명령으로 테스트를 실행한다. 32 | # python3 -m unittest -v 25_gabbi-flask.py 33 | def load_tests(loader, tests, pattern): 34 | return driver.build_tests(os.path.dirname(__file__), 35 | loader, intercept=lambda: application) 36 | 37 | 38 | if __name__ == "__main__": 39 | application.run() -------------------------------------------------------------------------------- /Chapter09/gabbi-flask.yaml: -------------------------------------------------------------------------------- 1 | tests: 2 | - name: GET root with If-Match match 3 | GET: / 4 | request_headers: 5 | If-Match: hword 6 | status: 200 7 | response_headers: 8 | ETag: hword 9 | 10 | - name: GET root with If-Match no match 11 | GET: / 12 | request_headers: 13 | If-Match: foobar 14 | status: 304 15 | response_forbidden_headers: 16 | - ETag 17 | 18 | - name: GET root with If-None-Match no match 19 | GET: / 20 | request_headers: 21 | If-None-Match: hword 22 | status: 304 23 | response_forbidden_headers: 24 | - ETag 25 | 26 | - name: GET root with If-None-Match match 27 | GET: / 28 | request_headers: 29 | If-None-Match: foobar 30 | status: 200 31 | response_headers: 32 | ETag: hword 33 | -------------------------------------------------------------------------------- /Chapter11/01_tox_shell/setup-memcached.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | clean_on_exit () { 4 | test -n "$MEMCACHED_DIR" && rm -rf "$MEMCACHED_DIR" 5 | } 6 | 7 | trap clean_on_exit EXIT 8 | 9 | wait_for_line () { 10 | while read line 11 | do 12 | echo "$line" | grep -q "$1" && break 13 | done < "$2" 14 | # 작업이 블록 되지 않도록, 계속 대기열을 읽어준다. 15 | cat "$2" >/dev/null & 16 | } 17 | 18 | MEMCACHED_DIR=`mktemp -d` 19 | mkfifo ${MEMCACHED_DIR}/out 20 | memcached -p 4526 -vv > ${MEMCACHED_DIR}/out 2>&1 & 21 | export MEMCACHED_PID=$! 22 | wait_for_line "server listening" ${MEMCACHED_DIR}/out 23 | 24 | $* 25 | 26 | kill ${MEMCACHED_PID} -------------------------------------------------------------------------------- /Chapter11/01_tox_shell/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | 4 | setup(name='test_memcached', 5 | version='0.1') -------------------------------------------------------------------------------- /Chapter11/01_tox_shell/test_memcached.py: -------------------------------------------------------------------------------- 1 | import os 2 | import socket 3 | import unittest 4 | 5 | 6 | class TestWithMemcached(unittest.TestCase): 7 | def setUp(self): 8 | super(TestWithMemcached, self).setUp() 9 | if not os.getenv("MEMCACHED_PID"): 10 | self.skipTest("Memcached is not running") 11 | 12 | def test_connect(self): 13 | s = socket.socket() 14 | s.connect(("localhost", 4526)) -------------------------------------------------------------------------------- /Chapter11/01_tox_shell/tox.ini: -------------------------------------------------------------------------------- 1 | [testenv] 2 | deps = nose 3 | commands=nosetests 4 | 5 | [testenv:py36-integration] 6 | commands=./setup-memcached.sh nosetests -------------------------------------------------------------------------------- /Chapter11/02_tox_pifpaf/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | 4 | setup(name='test_memcached', 5 | version='0.1') -------------------------------------------------------------------------------- /Chapter11/02_tox_pifpaf/test_memcached_pifpaf.py: -------------------------------------------------------------------------------- 1 | import os 2 | import socket 3 | import unittest 4 | 5 | 6 | class TestWithMemcached(unittest.TestCase): 7 | def setUp(self): 8 | super(TestWithMemcached, self).setUp() 9 | if not os.getenv("PIFPAF_MEMCACHED_URL"): 10 | self.skipTest("Memcached is not running") 11 | 12 | def test_connect(self): 13 | s = socket.socket() 14 | s.connect(("localhost", int(os.getenv("PIFPAF_MEMCACHED_PORT")))) 15 | -------------------------------------------------------------------------------- /Chapter11/02_tox_pifpaf/tox.ini: -------------------------------------------------------------------------------- 1 | [testenv] 2 | deps = nose 3 | pifpaf 4 | commands=nosetests 5 | 6 | [testenv:py36-integration] 7 | commands=pifpaf run memcached --port 19842 -- nosetests -------------------------------------------------------------------------------- /Chapter11/05_memcached-fixtures.py: -------------------------------------------------------------------------------- 1 | import socket 2 | 3 | import fixtures 4 | from pifpaf.drivers import memcached 5 | 6 | 7 | class TestWithMemcached(fixtures.TestWithFixtures): 8 | def setUp(self): 9 | super(TestWithMemcached, self).setUp() 10 | self.memcached = self.useFixture(memcached.MemcachedDriver(port=9742)) 11 | 12 | def test_connect(self): 13 | s = socket.socket() 14 | s.connect(("localhost", self.memcached.port)) -------------------------------------------------------------------------------- /Chapter11/06_app-memcached-fixtures.py: -------------------------------------------------------------------------------- 1 | import fixtures 2 | from pifpaf.drivers import memcached 3 | from pymemcache import client 4 | 5 | 6 | class AppException(Exception): 7 | pass 8 | 9 | 10 | class Application(object): 11 | def __init__(self, memcached=("localhost", 11211)): 12 | self.memcache = client.Client(memcached) 13 | 14 | def store_settings(self, settings): 15 | self.memcache.set("appsettings", settings) 16 | 17 | def retrieve_settings(self): 18 | return self.memcache.get("appsettings") 19 | 20 | 21 | class TestWithMemcached(fixtures.TestWithFixtures): 22 | def test_store_and_retrieve_settings(self): 23 | self.memcached = self.useFixture(memcached.MemcachedDriver(port=9742)) 24 | self.app = Application(("localhost", self.memcached.port)) 25 | self.app.store_settings(b"foobar") 26 | self.assertEqual(b"foobar", self.app.retrieve_settings()) -------------------------------------------------------------------------------- /Chapter11/07_app-memcached-fixtures-all.py: -------------------------------------------------------------------------------- 1 | import fixtures 2 | from pifpaf.drivers import memcached 3 | from pymemcache import client 4 | from pymemcache import exceptions 5 | 6 | 7 | class AppException(Exception): 8 | pass 9 | 10 | 11 | class Application(object): 12 | def __init__(self, memcached=("localhost", 11211)): 13 | self.memcache = client.Client(memcached) 14 | 15 | def store_settings(self, settings): 16 | try: 17 | self.memcache.set("appsettings", settings) 18 | except (exceptions.MemcacheError, 19 | ConnectionRefusedError, 20 | ConnectionResetError): 21 | raise AppException 22 | 23 | def retrieve_settings(self): 24 | try: 25 | return self.memcache.get("appsettings") 26 | except (exceptions.MemcacheError, 27 | ConnectionRefusedError, 28 | ConnectionResetError): 29 | raise AppException 30 | 31 | 32 | class TestWithMemcached(fixtures.TestWithFixtures): 33 | def test_store_and_retrieve_settings(self): 34 | self.memcached = self.useFixture(memcached.MemcachedDriver(port=9742)) 35 | self.app = Application(("localhost", self.memcached.port)) 36 | self.app.store_settings(b"foobar") 37 | self.assertEqual(b"foobar", self.app.retrieve_settings()) 38 | 39 | def test_connect_fail_on_store(self): 40 | self.app = Application(("localhost", 123)) 41 | self.assertRaises(AppException, 42 | self.app.store_settings, 43 | b"foobar") 44 | 45 | def test_connect_fail_on_retrieve(self): 46 | self.memcached = memcached.MemcachedDriver(port=9743) 47 | self.memcached.setUp() 48 | self.app = Application(("localhost", self.memcached.port)) 49 | self.app.store_settings(b"foobar") 50 | self.memcached.cleanUp() 51 | self.assertRaises(AppException, 52 | self.app.retrieve_settings) 53 | 54 | def test_memcached_restarted(self): 55 | self.memcached = memcached.MemcachedDriver(port=9744) 56 | self.memcached.setUp() 57 | self.app = Application(("localhost", self.memcached.port)) 58 | self.app.store_settings(b"foobar") 59 | self.memcached.reset() 60 | self.addCleanup(self.memcached.cleanUp) 61 | self.assertRaises(AppException, 62 | self.app.retrieve_settings) -------------------------------------------------------------------------------- /Chapter12/03_cache-webpage.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import cachetools 4 | import requests 5 | 6 | 7 | cache = cachetools.TTLCache(maxsize=5, ttl=5) 8 | URL = "http://httpbin.org/uuid" 9 | while True: 10 | try: 11 | print(cache[URL]) 12 | except KeyError: 13 | print("Paged not cached, fetching") 14 | page = requests.get("http://httpbin.org/uuid") 15 | cache[URL] = page.text 16 | print(page.text) 17 | time.sleep(1) -------------------------------------------------------------------------------- /Chapter12/07_memcached-connect.py: -------------------------------------------------------------------------------- 1 | from pymemcache.client import base 2 | 3 | # memcached는 이미 동작 중이어야 한다. 4 | client = base.Client(('localhost', 11211)) 5 | client.set('some_key', 'some_value') 6 | result = client.get('some_key') 7 | print(result) # 'some_value' 출력 -------------------------------------------------------------------------------- /Chapter12/08_memcached-missing-key.py: -------------------------------------------------------------------------------- 1 | from pymemcache.client import base 2 | 3 | 4 | def do_some_query(): 5 | # 실제로는 데이터베이스나 REST API로 원격에서 데이터를 가져온다고 가정함. 6 | return 42 7 | 8 | 9 | # memcached는 이미 동작 중이어야 한다. 10 | client = base.Client(('localhost', 11211)) 11 | result = client.get('some_key') 12 | if result is None: 13 | # 캐시에 없는 데이터는 원본 소스에서 가져와야 한다. 14 | result = do_some_query() 15 | # 다음 조회를 위해 결과를 캐시한다. 16 | client.set('some_key', result) 17 | print(result) -------------------------------------------------------------------------------- /Chapter12/09_pymemcache-fallback.py: -------------------------------------------------------------------------------- 1 | from pymemcache.client import base 2 | from pymemcache import fallback 3 | 4 | 5 | def do_some_query(): 6 | # 실제로는 데이터베이스나 REST API로 원격에서 데이터를 가져온다고 가정함. 7 | return 42 8 | 9 | 10 | # 'ignore_exc=True'로 설정해서 캐시 누락을 처리할 수 있도록 한다. 11 | # 새 캐시에 데이터가 채워지면, 이전 캐시 서버를 중지할 수 있다. 12 | old_cache = base.Client(('localhost', 11211), ignore_exc=True) 13 | new_cache = base.Client(('localhost', 11212)) 14 | 15 | client = fallback.FallbackClient((new_cache, old_cache)) 16 | 17 | result = client.get('some_key') 18 | if result is None: 19 | # 캐시에 없는 데이터는 원본 소스에서 가져와야 한다. 20 | result = do_some_query() 21 | # 다음 조회를 위해 결과를 캐시한다. 22 | client.set('some_key', result) 23 | print(result) 24 | -------------------------------------------------------------------------------- /Chapter12/10_memcached_user_count.py: -------------------------------------------------------------------------------- 1 | def on_visit(client): 2 | result = client.get('visitors') 3 | if result is None: 4 | result = 1 5 | else: 6 | result += 1 7 | client.set('visitors', result) -------------------------------------------------------------------------------- /Chapter12/11_memcached-CAS.py: -------------------------------------------------------------------------------- 1 | def on_visit(client): 2 | while True: 3 | result, cas = client.gets('visitors') 4 | if result is None: 5 | result = 1 6 | else: 7 | result += 1 8 | if client.cas('visitors', result, cas): 9 | break -------------------------------------------------------------------------------- /Chapter13/03_memory-view-copy.py: -------------------------------------------------------------------------------- 1 | @profile 2 | def read_random(): 3 | with open("./any_large_file", "rb") as source: 4 | content = source.read(1024 * 10000) 5 | content_to_write = content[1024:] 6 | print("Content length: %d, content to write length %d" % 7 | (len(content), len(content_to_write))) 8 | with open("/dev/null", "wb") as target: 9 | target.write(content_to_write) 10 | 11 | if __name__ == '__main__': 12 | read_random() -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | ### (번역서) 실전 스케일링 파이썬 프로그래밍 3 | 4 | ![](/cover.jpg) 5 | 6 | * 출간일: 2018년 8월 21일 7 | * 각 챕터별 예제 코드가 들어 있습니다. 8 | * 책과 관련된 문의는 corecode.pe.kr 도서 페이지에 남겨 주세요. 9 | * 도서 정보 10 | * [예스24](http://www.yes24.com/24/goods/63747074?scode=032&OzSrank=1) 11 | * [알라딘](http://www.aladin.co.kr/shop/wproduct.aspx?ItemId=163437128) 12 | -------------------------------------------------------------------------------- /cover.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/surinkim/scaling_python_kor/5f65175a450e2380b9a8d6c3f5a06c38ec2cb55e/cover.jpg --------------------------------------------------------------------------------