├── .gitignore ├── LICENSE ├── README.md ├── concorrencia ├── .gitignore ├── async.odp ├── async.pdf ├── async │ ├── README.rst │ ├── domains │ │ ├── README.rst │ │ ├── asyncio │ │ │ ├── blogdom.py │ │ │ ├── blogdom_sec.py │ │ │ ├── domaincheck.py │ │ │ └── domainlib.py │ │ └── curio │ │ │ ├── blogdom.py │ │ │ ├── domaincheck.py │ │ │ ├── domainlib.py │ │ │ └── requirements.txt │ └── mojifinder │ │ ├── .fastapi │ │ ├── bin │ │ │ ├── Activate.ps1 │ │ │ ├── activate │ │ │ ├── activate.csh │ │ │ ├── activate.fish │ │ │ ├── pip │ │ │ ├── pip3 │ │ │ ├── pip3.11 │ │ │ ├── python │ │ │ ├── python3 │ │ │ ├── python3.11 │ │ │ └── uvicorn │ │ └── pyvenv.cfg │ │ ├── README.md │ │ ├── bottle.py │ │ ├── charindex.py │ │ ├── requirements.txt │ │ ├── static │ │ └── form.html │ │ ├── tcp_mojifinder.py │ │ ├── web_mojifinder.py │ │ ├── web_mojifinder.sh │ │ └── web_mojifinder_bottle.py ├── clock.py ├── clock_face.py ├── concorrencia-x-paralelismo.odp ├── concorrencia-x-paralelismo.pdf ├── curio-example │ ├── curio-example.ipynb │ └── curio │ │ ├── README.md │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── channel.py │ │ ├── debug.py │ │ ├── errors.py │ │ ├── file.py │ │ ├── io.py │ │ ├── kernel.py │ │ ├── meta.py │ │ ├── monitor.py │ │ ├── network.py │ │ ├── queue.py │ │ ├── sched.py │ │ ├── socket.py │ │ ├── ssl.py │ │ ├── sync.py │ │ ├── task.py │ │ ├── thread.py │ │ ├── time.py │ │ ├── timequeue.py │ │ ├── traps.py │ │ └── workers.py ├── do-zero-pronto │ ├── blogdom_asyncio.py │ ├── blogdom_curio.py │ ├── blogdom_trio.py │ ├── curio │ ├── gira_async.py │ ├── gira_proc.py │ ├── gira_sec.py │ ├── gira_sec_braille.py │ └── gira_thread.py ├── do-zero │ ├── gira_proc.py │ ├── notebook_components.png │ ├── ola-mundo-concorrente-pronto.ipynb │ ├── ola-mundo-concorrente.ipynb │ ├── pcworld-sel.jpg │ └── tty33.jpg ├── executors │ ├── demo_executor_map.py │ ├── getflags │ │ ├── .gitignore │ │ ├── README.adoc │ │ ├── country_codes.txt │ │ ├── flags.py │ │ ├── flags.zip │ │ ├── flags2_asyncio.py │ │ ├── flags2_asyncio_executor.py │ │ ├── flags2_common.py │ │ ├── flags2_sequential.py │ │ ├── flags2_threadpool.py │ │ ├── flags3_asyncio.py │ │ ├── flags_asyncio.py │ │ ├── flags_threadpool.py │ │ ├── flags_threadpool_futures.py │ │ ├── httpx-error-tree │ │ │ ├── drawtree.py │ │ │ └── tree.py │ │ ├── requirements.txt │ │ └── slow_server.py │ └── primes │ │ ├── primes.py │ │ └── proc_pool.py ├── fio.py ├── giro.py ├── modelos-de-concorrencia.odp ├── modelos-de-concorrencia.pdf ├── primes │ ├── .gitignore │ ├── 1_primo.py │ ├── 1_primo_async_NO_SPIN.py │ ├── 1_primo_proc_spin.py │ ├── 1_primo_thread_spin.py │ ├── README.md │ ├── future_procs.py │ ├── future_threads.py │ ├── integers.py │ ├── map_primes.py │ ├── n_primes_proc.ipynb │ ├── n_primes_proc.py │ ├── prime-cookies.png │ ├── primes-demo.ipynb │ ├── primes.py │ ├── primeserver │ │ ├── go.mod │ │ └── server.go │ ├── process-workflow.png │ ├── table.py │ ├── time_primes.py │ └── widget-lab.ipynb ├── processos-e-threads.odp ├── processos-e-threads.pdf ├── spin_proc.py └── wikipics │ ├── README.md │ ├── coro_1_download.py │ ├── de-gera-a-coro.ipynb │ ├── future_thread_downloads.py │ ├── gallery.py │ ├── img │ ├── .gitignore │ └── README.md │ ├── jpeg-by-size.txt │ ├── no-image.png │ ├── process_1_download.py │ ├── spinner.py │ ├── thread_1_download.py │ ├── wiki_probe_time.py │ ├── wikipics-async-demo.ipynb │ ├── wikipics-demo.ipynb │ └── wikipics.py ├── data-model ├── README.rst ├── frenchdeck.doctest ├── frenchdeck.py ├── frenchdeck_soln.ipynb ├── modelo-de-dados-1.ipynb ├── vector2d.ipynb ├── vector2d.py └── vector2d_soln.ipynb ├── intro └── notebook_components.webp ├── memoria ├── fig2-1.png ├── fig2-4.png ├── fig6-3.png ├── fig6-4.png ├── memoria-p1.ipynb ├── memoria-p2.ipynb ├── memoria-p3.ipynb ├── memoria-p4.ipynb ├── memoria-p5.ipynb ├── memoria.ipynb └── var-boxes-x-labels.png ├── pyproject.toml └── tipos ├── columnize.py ├── coord_nt.py ├── coord_tuple.py ├── double.py ├── messages.py ├── messages_tests.py ├── mode_hashable.py ├── mypy.ini ├── replacer.py └── sample.py /.gitignore: -------------------------------------------------------------------------------- 1 | .py3??/ 2 | .vscode/ 3 | 4 | .DS_Store 5 | 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | 11 | # C extensions 12 | *.so 13 | 14 | # Distribution / packaging 15 | .Python 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | downloads/ 20 | eggs/ 21 | .eggs/ 22 | lib/ 23 | lib64/ 24 | parts/ 25 | sdist/ 26 | var/ 27 | wheels/ 28 | pip-wheel-metadata/ 29 | share/python-wheels/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | MANIFEST 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .nox/ 49 | .coverage 50 | .coverage.* 51 | .cache 52 | nosetests.xml 53 | coverage.xml 54 | *.cover 55 | *.py,cover 56 | .hypothesis/ 57 | .pytest_cache/ 58 | 59 | # Translations 60 | *.mo 61 | *.pot 62 | 63 | # Django stuff: 64 | *.log 65 | local_settings.py 66 | db.sqlite3 67 | db.sqlite3-journal 68 | 69 | # Flask stuff: 70 | instance/ 71 | .webassets-cache 72 | 73 | # Scrapy stuff: 74 | .scrapy 75 | 76 | # Sphinx documentation 77 | docs/_build/ 78 | 79 | # PyBuilder 80 | target/ 81 | 82 | # Jupyter Notebook 83 | .ipynb_checkpoints 84 | 85 | # IPython 86 | profile_default/ 87 | ipython_config.py 88 | 89 | # pyenv 90 | .python-version 91 | 92 | # pipenv 93 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 94 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 95 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 96 | # install all needed dependencies. 97 | #Pipfile.lock 98 | 99 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 100 | __pypackages__/ 101 | 102 | # Celery stuff 103 | celerybeat-schedule 104 | celerybeat.pid 105 | 106 | # SageMath parsed files 107 | *.sage.py 108 | 109 | # Environments 110 | .env 111 | .venv 112 | env/ 113 | venv/ 114 | ENV/ 115 | env.bak/ 116 | venv.bak/ 117 | 118 | # Spyder project settings 119 | .spyderproject 120 | .spyproject 121 | 122 | # Rope project settings 123 | .ropeproject 124 | 125 | # mkdocs documentation 126 | /site 127 | 128 | # mypy 129 | .mypy_cache/ 130 | .dmypy.json 131 | dmypy.json 132 | 133 | # Pyre type checker 134 | .pyre/ 135 | 136 | concorrencia/primes/primeserver/primeserver 137 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Luciano Ramalho 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # python-eng 2 | Material para o curso Python Engineer da Linux Tips 3 | 4 | ## Aulas 5 | 6 | 0. Introdução ao curso e ferramentas 7 | 1. Memória 8 | 2. Anotações de tipo em funções 9 | 10 | 11 | -------------------------------------------------------------------------------- /concorrencia/.gitignore: -------------------------------------------------------------------------------- 1 | .py3??/ 2 | .vscode/ 3 | 4 | *.bkp 5 | .DS_Store 6 | 7 | # Arquivos renderizados 8 | *.html 9 | 10 | # Byte-compiled / optimized / DLL files 11 | __pycache__/ 12 | *.py[cod] 13 | *$py.class 14 | 15 | # C extensions 16 | *.so 17 | 18 | # Distribution / packaging 19 | .Python 20 | build/ 21 | develop-eggs/ 22 | dist/ 23 | downloads/ 24 | eggs/ 25 | .eggs/ 26 | lib/ 27 | lib64/ 28 | parts/ 29 | sdist/ 30 | var/ 31 | wheels/ 32 | pip-wheel-metadata/ 33 | share/python-wheels/ 34 | *.egg-info/ 35 | .installed.cfg 36 | *.egg 37 | MANIFEST 38 | 39 | # PyInstaller 40 | # Usually these files are written by a python script from a template 41 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 42 | *.manifest 43 | *.spec 44 | 45 | # Installer logs 46 | pip-log.txt 47 | pip-delete-this-directory.txt 48 | 49 | # Unit test / coverage reports 50 | htmlcov/ 51 | .tox/ 52 | .nox/ 53 | .coverage 54 | .coverage.* 55 | .cache 56 | nosetests.xml 57 | coverage.xml 58 | *.cover 59 | *.py,cover 60 | .hypothesis/ 61 | .pytest_cache/ 62 | 63 | # Translations 64 | *.mo 65 | *.pot 66 | 67 | # Django stuff: 68 | *.log 69 | local_settings.py 70 | db.sqlite3 71 | db.sqlite3-journal 72 | 73 | # Flask stuff: 74 | instance/ 75 | .webassets-cache 76 | 77 | # Scrapy stuff: 78 | .scrapy 79 | 80 | # Sphinx documentation 81 | docs/_build/ 82 | 83 | # PyBuilder 84 | target/ 85 | 86 | # Jupyter Notebook 87 | .ipynb_checkpoints 88 | 89 | # IPython 90 | profile_default/ 91 | ipython_config.py 92 | 93 | # pyenv 94 | .python-version 95 | 96 | # pipenv 97 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 98 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 99 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 100 | # install all needed dependencies. 101 | #Pipfile.lock 102 | 103 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 104 | __pypackages__/ 105 | 106 | # Celery stuff 107 | celerybeat-schedule 108 | celerybeat.pid 109 | 110 | # SageMath parsed files 111 | *.sage.py 112 | 113 | # Environments 114 | .env 115 | .venv 116 | env/ 117 | venv/ 118 | ENV/ 119 | env.bak/ 120 | venv.bak/ 121 | 122 | # Spyder project settings 123 | .spyderproject 124 | .spyproject 125 | 126 | # Rope project settings 127 | .ropeproject 128 | 129 | # mkdocs documentation 130 | /site 131 | 132 | # mypy 133 | .mypy_cache/ 134 | .dmypy.json 135 | dmypy.json 136 | 137 | # Pyre type checker 138 | .pyre/ -------------------------------------------------------------------------------- /concorrencia/async.odp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramalho/python-eng/59a007d54595f11441ec7de07909576e1c158772/concorrencia/async.odp -------------------------------------------------------------------------------- /concorrencia/async.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramalho/python-eng/59a007d54595f11441ec7de07909576e1c158772/concorrencia/async.pdf -------------------------------------------------------------------------------- /concorrencia/async/README.rst: -------------------------------------------------------------------------------- 1 | Sample code for Chapter 22 - "Asynchronous programming" 2 | 3 | From the book "Fluent Python, Second Edition" by Luciano Ramalho (O'Reilly, 2021) 4 | https://learning.oreilly.com/library/view/fluent-python-2nd/9781492056348/ 5 | -------------------------------------------------------------------------------- /concorrencia/async/domains/README.rst: -------------------------------------------------------------------------------- 1 | domainlib demonstration 2 | ======================= 3 | 4 | Run Python's async console (requires Python ≥ 3.8):: 5 | 6 | $ python3 -m asyncio 7 | 8 | I'll see ``asyncio`` imported automatically:: 9 | 10 | >>> import asyncio 11 | 12 | Now you can experiment with ``domainlib``. 13 | 14 | At the `>>>` prompt, type these commands:: 15 | 16 | >>> from domainlib import * 17 | >>> await probe('python.org') 18 | 19 | Note the result. 20 | 21 | Next:: 22 | 23 | >>> names = 'python.org rust-lang.org golang.org n05uch1an9.org'.split() 24 | >>> async for result in multi_probe(names): 25 | ... print(*result, sep='\t') 26 | 27 | Note that if you run the last two lines again, 28 | the results are likely to appear in a different order. 29 | -------------------------------------------------------------------------------- /concorrencia/async/domains/asyncio/blogdom.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from asyncio import run, get_running_loop, as_completed 3 | import socket 4 | from keyword import kwlist, softkwlist 5 | 6 | MAX_KEYWORD_LEN = 10 # <1> 7 | KEYWORDS = sorted(kwlist + softkwlist) 8 | 9 | async def probe(domain: str) -> tuple[str, bool]: # <2> 10 | loop = get_running_loop() # <3> 11 | try: 12 | await loop.getaddrinfo(domain, None) # <4> 13 | except socket.gaierror: 14 | return (domain, False) 15 | return (domain, True) 16 | 17 | async def main() -> None: # <5> 18 | names = (kw for kw in KEYWORDS if len(kw) <= MAX_KEYWORD_LEN) # <6> 19 | domains = (f'{name}.dev'.lower() for name in names) # <7> 20 | coros = [probe(domain) for domain in sorted(domains)] # <8> 21 | for coro in as_completed(coros): # <9> 22 | domain, found = await coro # <10> 23 | mark = '+' if found else ' ' 24 | print(f'{mark} {domain}') 25 | 26 | 27 | if __name__ == '__main__': 28 | run(main()) # <11> 29 | -------------------------------------------------------------------------------- /concorrencia/async/domains/asyncio/blogdom_sec.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from asyncio import run, get_running_loop, as_completed 3 | import socket 4 | from keyword import kwlist, softkwlist 5 | 6 | MAX_KEYWORD_LEN = 10 # <1> 7 | KEYWORDS = sorted(kwlist + softkwlist) 8 | 9 | async def probe(domain: str) -> tuple[str, bool]: # <2> 10 | loop = get_running_loop() # <3> 11 | try: 12 | await loop.getaddrinfo(domain, None) # <4> 13 | except socket.gaierror: 14 | return (domain, False) 15 | return (domain, True) 16 | 17 | 18 | def main() -> None: # <5> 19 | names = (kw for kw in KEYWORDS if len(kw) <= MAX_KEYWORD_LEN) # <6> 20 | domains = (f'{name}.dev'.lower() for name in names) # <7> 21 | for domain in sorted(domains): # <9> 22 | domain, found = run(probe(domain)) 23 | mark = '+' if found else ' ' 24 | print(f'{mark} {domain}') 25 | 26 | 27 | if __name__ == '__main__': 28 | main() # <11> 29 | -------------------------------------------------------------------------------- /concorrencia/async/domains/asyncio/domaincheck.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import asyncio 3 | import sys 4 | from keyword import kwlist 5 | 6 | from domainlib import multi_probe 7 | 8 | 9 | async def main(tld: str) -> None: 10 | tld = tld.strip('.') 11 | names = (kw for kw in kwlist if len(kw) <= 4) 12 | domains = (f'{name}.{tld}'.lower() for name in names) 13 | print('FOUND\t\tNOT FOUND') 14 | print('=====\t\t=========') 15 | async for domain, found in multi_probe(domains): 16 | indent = '' if found else '\t\t' 17 | print(f'{indent}{domain}') 18 | 19 | 20 | if __name__ == '__main__': 21 | if len(sys.argv) == 2: 22 | asyncio.run(main(sys.argv[1])) 23 | else: 24 | print('Please provide a TLD.', f'Example: {sys.argv[0]} COM.BR') 25 | -------------------------------------------------------------------------------- /concorrencia/async/domains/asyncio/domainlib.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Iterable, AsyncIterator 2 | from typing import NamedTuple 3 | 4 | import asyncio 5 | import socket 6 | 7 | 8 | class Result(NamedTuple): # <1> 9 | domain: str 10 | found: bool 11 | 12 | 13 | async def probe(domain: str, loop: asyncio.AbstractEventLoop) -> Result: # <3> 14 | try: 15 | await loop.getaddrinfo(domain, None) 16 | except socket.gaierror: 17 | return Result(domain, False) 18 | return Result(domain, True) 19 | 20 | 21 | async def multi_probe(domains: Iterable[str]) -> AsyncIterator[Result]: # <4> 22 | loop = asyncio.get_running_loop() 23 | coros = [probe(domain, loop) for domain in domains] # <5> 24 | for coro in asyncio.as_completed(coros): # <6> 25 | result = await coro # <7> 26 | yield result # <8> 27 | -------------------------------------------------------------------------------- /concorrencia/async/domains/curio/blogdom.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from curio import run, TaskGroup 3 | import curio.socket as socket 4 | from keyword import kwlist, softkwlist 5 | 6 | MAX_KEYWORD_LEN = 5 7 | KEYWORDS = sorted(kwlist + softkwlist) 8 | 9 | async def probe(domain: str) -> tuple[str, bool]: # <1> 10 | try: 11 | await socket.getaddrinfo(domain, None) # <2> 12 | except socket.gaierror: 13 | return (domain, False) 14 | return (domain, True) 15 | 16 | async def main() -> None: 17 | names = (kw for kw in kwlist if len(kw) <= MAX_KEYWORD_LEN) 18 | domains = (f'{name}.dev'.lower() for name in names) 19 | async with TaskGroup() as group: # <3> 20 | for domain in domains: 21 | await group.spawn(probe, domain) # <4> 22 | async for task in group: # <5> 23 | domain, found = task.result 24 | mark = '+' if found else ' ' 25 | print(f'{mark} {domain}') 26 | 27 | if __name__ == '__main__': 28 | run(main()) # <6> 29 | -------------------------------------------------------------------------------- /concorrencia/async/domains/curio/domaincheck.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import curio 3 | import sys 4 | from keyword import kwlist 5 | 6 | from domainlib import multi_probe 7 | 8 | 9 | async def main(tld: str) -> None: 10 | tld = tld.strip('.') 11 | names = (kw for kw in kwlist if len(kw) <= 4) 12 | domains = (f'{name}.{tld}'.lower() for name in names) 13 | print('FOUND\t\tNOT FOUND') 14 | print('=====\t\t=========') 15 | async for domain, found in multi_probe(domains): 16 | indent = '' if found else '\t\t' 17 | print(f'{indent}{domain}') 18 | 19 | 20 | if __name__ == '__main__': 21 | if len(sys.argv) == 2: 22 | curio.run(main(sys.argv[1])) 23 | else: 24 | print('Please provide a TLD.', f'Example: {sys.argv[0]} COM.BR') 25 | -------------------------------------------------------------------------------- /concorrencia/async/domains/curio/domainlib.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Iterable, AsyncIterator 2 | from typing import NamedTuple 3 | 4 | from curio import TaskGroup 5 | import curio.socket as socket 6 | 7 | 8 | class Result(NamedTuple): 9 | domain: str 10 | found: bool 11 | 12 | 13 | async def probe(domain: str) -> Result: 14 | try: 15 | await socket.getaddrinfo(domain, None) 16 | except socket.gaierror: 17 | return Result(domain, False) 18 | return Result(domain, True) 19 | 20 | 21 | async def multi_probe(domains: Iterable[str]) -> AsyncIterator[Result]: 22 | async with TaskGroup() as group: 23 | for domain in domains: 24 | await group.spawn(probe, domain) 25 | async for task in group: 26 | yield task.result 27 | -------------------------------------------------------------------------------- /concorrencia/async/domains/curio/requirements.txt: -------------------------------------------------------------------------------- 1 | curio==1.5 2 | -------------------------------------------------------------------------------- /concorrencia/async/mojifinder/.fastapi/bin/activate: -------------------------------------------------------------------------------- 1 | # This file must be used with "source bin/activate" *from bash* 2 | # you cannot run it directly 3 | 4 | deactivate () { 5 | # reset old environment variables 6 | if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then 7 | PATH="${_OLD_VIRTUAL_PATH:-}" 8 | export PATH 9 | unset _OLD_VIRTUAL_PATH 10 | fi 11 | if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then 12 | PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}" 13 | export PYTHONHOME 14 | unset _OLD_VIRTUAL_PYTHONHOME 15 | fi 16 | 17 | # This should detect bash and zsh, which have a hash command that must 18 | # be called to get it to forget past commands. Without forgetting 19 | # past commands the $PATH changes we made may not be respected 20 | if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then 21 | hash -r 2> /dev/null 22 | fi 23 | 24 | if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then 25 | PS1="${_OLD_VIRTUAL_PS1:-}" 26 | export PS1 27 | unset _OLD_VIRTUAL_PS1 28 | fi 29 | 30 | unset VIRTUAL_ENV 31 | unset VIRTUAL_ENV_PROMPT 32 | if [ ! "${1:-}" = "nondestructive" ] ; then 33 | # Self destruct! 34 | unset -f deactivate 35 | fi 36 | } 37 | 38 | # unset irrelevant variables 39 | deactivate nondestructive 40 | 41 | VIRTUAL_ENV="/Users/luciano/prj/linuxtips/python-eng/async/mojifinder/.fastapi" 42 | export VIRTUAL_ENV 43 | 44 | _OLD_VIRTUAL_PATH="$PATH" 45 | PATH="$VIRTUAL_ENV/bin:$PATH" 46 | export PATH 47 | 48 | # unset PYTHONHOME if set 49 | # this will fail if PYTHONHOME is set to the empty string (which is bad anyway) 50 | # could use `if (set -u; : $PYTHONHOME) ;` in bash 51 | if [ -n "${PYTHONHOME:-}" ] ; then 52 | _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}" 53 | unset PYTHONHOME 54 | fi 55 | 56 | if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then 57 | _OLD_VIRTUAL_PS1="${PS1:-}" 58 | PS1="(.fastapi) ${PS1:-}" 59 | export PS1 60 | VIRTUAL_ENV_PROMPT="(.fastapi) " 61 | export VIRTUAL_ENV_PROMPT 62 | fi 63 | 64 | # This should detect bash and zsh, which have a hash command that must 65 | # be called to get it to forget past commands. Without forgetting 66 | # past commands the $PATH changes we made may not be respected 67 | if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then 68 | hash -r 2> /dev/null 69 | fi 70 | -------------------------------------------------------------------------------- /concorrencia/async/mojifinder/.fastapi/bin/activate.csh: -------------------------------------------------------------------------------- 1 | # This file must be used with "source bin/activate.csh" *from csh*. 2 | # You cannot run it directly. 3 | # Created by Davide Di Blasi . 4 | # Ported to Python 3.3 venv by Andrew Svetlov 5 | 6 | alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; unsetenv VIRTUAL_ENV_PROMPT; test "\!:*" != "nondestructive" && unalias deactivate' 7 | 8 | # Unset irrelevant variables. 9 | deactivate nondestructive 10 | 11 | setenv VIRTUAL_ENV "/Users/luciano/prj/linuxtips/python-eng/async/mojifinder/.fastapi" 12 | 13 | set _OLD_VIRTUAL_PATH="$PATH" 14 | setenv PATH "$VIRTUAL_ENV/bin:$PATH" 15 | 16 | 17 | set _OLD_VIRTUAL_PROMPT="$prompt" 18 | 19 | if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then 20 | set prompt = "(.fastapi) $prompt" 21 | setenv VIRTUAL_ENV_PROMPT "(.fastapi) " 22 | endif 23 | 24 | alias pydoc python -m pydoc 25 | 26 | rehash 27 | -------------------------------------------------------------------------------- /concorrencia/async/mojifinder/.fastapi/bin/activate.fish: -------------------------------------------------------------------------------- 1 | # This file must be used with "source /bin/activate.fish" *from fish* 2 | # (https://fishshell.com/); you cannot run it directly. 3 | 4 | function deactivate -d "Exit virtual environment and return to normal shell environment" 5 | # reset old environment variables 6 | if test -n "$_OLD_VIRTUAL_PATH" 7 | set -gx PATH $_OLD_VIRTUAL_PATH 8 | set -e _OLD_VIRTUAL_PATH 9 | end 10 | if test -n "$_OLD_VIRTUAL_PYTHONHOME" 11 | set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME 12 | set -e _OLD_VIRTUAL_PYTHONHOME 13 | end 14 | 15 | if test -n "$_OLD_FISH_PROMPT_OVERRIDE" 16 | set -e _OLD_FISH_PROMPT_OVERRIDE 17 | # prevents error when using nested fish instances (Issue #93858) 18 | if functions -q _old_fish_prompt 19 | functions -e fish_prompt 20 | functions -c _old_fish_prompt fish_prompt 21 | functions -e _old_fish_prompt 22 | end 23 | end 24 | 25 | set -e VIRTUAL_ENV 26 | set -e VIRTUAL_ENV_PROMPT 27 | if test "$argv[1]" != "nondestructive" 28 | # Self-destruct! 29 | functions -e deactivate 30 | end 31 | end 32 | 33 | # Unset irrelevant variables. 34 | deactivate nondestructive 35 | 36 | set -gx VIRTUAL_ENV "/Users/luciano/prj/linuxtips/python-eng/async/mojifinder/.fastapi" 37 | 38 | set -gx _OLD_VIRTUAL_PATH $PATH 39 | set -gx PATH "$VIRTUAL_ENV/bin" $PATH 40 | 41 | # Unset PYTHONHOME if set. 42 | if set -q PYTHONHOME 43 | set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME 44 | set -e PYTHONHOME 45 | end 46 | 47 | if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" 48 | # fish uses a function instead of an env var to generate the prompt. 49 | 50 | # Save the current fish_prompt function as the function _old_fish_prompt. 51 | functions -c fish_prompt _old_fish_prompt 52 | 53 | # With the original prompt function renamed, we can override with our own. 54 | function fish_prompt 55 | # Save the return status of the last command. 56 | set -l old_status $status 57 | 58 | # Output the venv prompt; color taken from the blue of the Python logo. 59 | printf "%s%s%s" (set_color 4B8BBE) "(.fastapi) " (set_color normal) 60 | 61 | # Restore the return status of the previous command. 62 | echo "exit $old_status" | . 63 | # Output the original/"old" prompt. 64 | _old_fish_prompt 65 | end 66 | 67 | set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" 68 | set -gx VIRTUAL_ENV_PROMPT "(.fastapi) " 69 | end 70 | -------------------------------------------------------------------------------- /concorrencia/async/mojifinder/.fastapi/bin/pip: -------------------------------------------------------------------------------- 1 | #!/Users/luciano/prj/linuxtips/python-eng/async/mojifinder/.fastapi/bin/python3.11 2 | # -*- coding: utf-8 -*- 3 | import re 4 | import sys 5 | from pip._internal.cli.main import main 6 | if __name__ == '__main__': 7 | sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) 8 | sys.exit(main()) 9 | -------------------------------------------------------------------------------- /concorrencia/async/mojifinder/.fastapi/bin/pip3: -------------------------------------------------------------------------------- 1 | #!/Users/luciano/prj/linuxtips/python-eng/async/mojifinder/.fastapi/bin/python3.11 2 | # -*- coding: utf-8 -*- 3 | import re 4 | import sys 5 | from pip._internal.cli.main import main 6 | if __name__ == '__main__': 7 | sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) 8 | sys.exit(main()) 9 | -------------------------------------------------------------------------------- /concorrencia/async/mojifinder/.fastapi/bin/pip3.11: -------------------------------------------------------------------------------- 1 | #!/Users/luciano/prj/linuxtips/python-eng/async/mojifinder/.fastapi/bin/python3.11 2 | # -*- coding: utf-8 -*- 3 | import re 4 | import sys 5 | from pip._internal.cli.main import main 6 | if __name__ == '__main__': 7 | sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) 8 | sys.exit(main()) 9 | -------------------------------------------------------------------------------- /concorrencia/async/mojifinder/.fastapi/bin/python: -------------------------------------------------------------------------------- 1 | python3.11 -------------------------------------------------------------------------------- /concorrencia/async/mojifinder/.fastapi/bin/python3: -------------------------------------------------------------------------------- 1 | python3.11 -------------------------------------------------------------------------------- /concorrencia/async/mojifinder/.fastapi/bin/python3.11: -------------------------------------------------------------------------------- 1 | /Library/Frameworks/Python.framework/Versions/3.11/bin/python3.11 -------------------------------------------------------------------------------- /concorrencia/async/mojifinder/.fastapi/bin/uvicorn: -------------------------------------------------------------------------------- 1 | #!/Users/luciano/prj/linuxtips/python-eng/async/mojifinder/.fastapi/bin/python3.11 2 | # -*- coding: utf-8 -*- 3 | import re 4 | import sys 5 | from uvicorn.main import main 6 | if __name__ == '__main__': 7 | sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) 8 | sys.exit(main()) 9 | -------------------------------------------------------------------------------- /concorrencia/async/mojifinder/.fastapi/pyvenv.cfg: -------------------------------------------------------------------------------- 1 | home = /Library/Frameworks/Python.framework/Versions/3.11/bin 2 | include-system-site-packages = false 3 | version = 3.11.1 4 | executable = /Library/Frameworks/Python.framework/Versions/3.11/bin/python3.11 5 | command = /Library/Frameworks/Python.framework/Versions/3.11/bin/python3 -m venv /Users/luciano/prj/linuxtips/python-eng/async/mojifinder/.fastapi 6 | -------------------------------------------------------------------------------- /concorrencia/async/mojifinder/README.md: -------------------------------------------------------------------------------- 1 | # Mojifinder: Unicode character search examples 2 | 3 | Examples from _Fluent Python, Second Edition_—Chapter 22, _Asynchronous Programming_. 4 | 5 | ## How to run `web_mojifinder.py` 6 | 7 | `web_mojifinder.py` is a Web application built with _[FastAPI](https://fastapi.tiangolo.com/)_. 8 | To run it, first install _FastAPI_ and an ASGI server. 9 | The application was tested with _[Uvicorn](https://www.uvicorn.org/)_. 10 | 11 | ``` 12 | $ pip install fastapi uvicorn 13 | ``` 14 | 15 | Now you can use `uvicorn` to run the app. 16 | 17 | ``` 18 | $ uvicorn web_mojifinder:app 19 | ``` 20 | 21 | Finally, visit http://127.0.0.1:8000/ with your browser to see the search form. 22 | 23 | 24 | ## Directory contents 25 | 26 | These files can be run as scripts directly from the command line: 27 | 28 | - `charindex.py`: libray used by the Mojifinder examples. Also works as CLI search script. 29 | - `tcp_mojifinder.py`: TCP/IP Unicode search server. Depends only on the Python 3.9 standard library. Use a telnet application as client. 30 | - `web_mojifinder_bottle.py`: Unicode Web service. Depends on `bottle.py` and `static/form.html`. Use an HTTP browser as client. 31 | 32 | This program requires an ASGI server to run it: 33 | 34 | - `web_mojifinder.py`: Unicode Web service. Depends on _[FastAPI](https://fastapi.tiangolo.com/)_ and `static/form.html`. 35 | 36 | Support files: 37 | 38 | - `bottle.py`: local copy of the single-file _[Bottle](https://bottlepy.org/)_ Web framework. 39 | - `requirements.txt`: list of dependencies for `web_mojifinder.py`. 40 | - `static/form.html`: HTML form used by the `web_*` examples. 41 | - `README.md`: this file 🤓 42 | -------------------------------------------------------------------------------- /concorrencia/async/mojifinder/charindex.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Class ``InvertedIndex`` builds an inverted index mapping each word to 5 | the set of Unicode characters which contain that word in their names. 6 | 7 | Optional arguments to the constructor are ``first`` and ``last+1`` 8 | character codes to index, to make testing easier. In the examples 9 | below, only the ASCII range was indexed. 10 | 11 | The `entries` attribute is a `defaultdict` with uppercased single 12 | words as keys:: 13 | 14 | >>> idx = InvertedIndex(32, 128) 15 | >>> idx.entries['DOLLAR'] 16 | {'$'} 17 | >>> sorted(idx.entries['SIGN']) 18 | ['#', '$', '%', '+', '<', '=', '>'] 19 | >>> idx.entries['A'] & idx.entries['SMALL'] 20 | {'a'} 21 | >>> idx.entries['BRILLIG'] 22 | set() 23 | 24 | The `.search()` method takes a string, uppercases it, splits it into 25 | words, and returns the intersection of the entries for each word:: 26 | 27 | >>> idx.search('capital a') 28 | {'A'} 29 | 30 | """ 31 | 32 | import sys 33 | import unicodedata 34 | from collections import defaultdict 35 | from collections.abc import Iterator 36 | 37 | STOP_CODE: int = sys.maxunicode + 1 38 | 39 | Char = str 40 | Index = defaultdict[str, set[Char]] 41 | 42 | 43 | def tokenize(text: str) -> Iterator[str]: 44 | """return iterator of uppercased words""" 45 | for word in text.upper().replace('-', ' ').split(): 46 | yield word 47 | 48 | 49 | class InvertedIndex: 50 | entries: Index 51 | 52 | def __init__(self, start: int = 32, stop: int = STOP_CODE): 53 | entries: Index = defaultdict(set) 54 | for char in (chr(i) for i in range(start, stop)): 55 | name = unicodedata.name(char, '') 56 | if name: 57 | for word in tokenize(name): 58 | entries[word].add(char) 59 | self.entries = entries 60 | 61 | def search(self, query: str) -> set[Char]: 62 | if words := list(tokenize(query)): 63 | found = self.entries[words[0]] 64 | return found.intersection(*(self.entries[w] for w in words[1:])) 65 | else: 66 | return set() 67 | 68 | 69 | def format_results(chars: set[Char]) -> Iterator[str]: 70 | for char in sorted(chars): 71 | name = unicodedata.name(char) 72 | code = ord(char) 73 | yield f'U+{code:04X}\t{char}\t{name}' 74 | 75 | 76 | def main(words: list[str]) -> None: 77 | if not words: 78 | print('Please give one or more words to search.') 79 | sys.exit(2) # command line usage error 80 | index = InvertedIndex() 81 | chars = index.search(' '.join(words)) 82 | for line in format_results(chars): 83 | print(line) 84 | print('─' * 66, f'{len(chars)} found') 85 | 86 | 87 | if __name__ == '__main__': 88 | main(sys.argv[1:]) 89 | -------------------------------------------------------------------------------- /concorrencia/async/mojifinder/requirements.txt: -------------------------------------------------------------------------------- 1 | anyio==3.7.0 2 | click==8.1.3 3 | fastapi==0.98.0 4 | h11==0.14.0 5 | idna==3.4 6 | pydantic==1.10.9 7 | sniffio==1.3.0 8 | starlette==0.27.0 9 | typing_extensions==4.6.3 10 | uvicorn==0.22.0 11 | -------------------------------------------------------------------------------- /concorrencia/async/mojifinder/static/form.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Mojifinder 6 | 14 | 70 | 71 | 72 | 73 |
74 | 75 | 76 |
77 | 78 | 79 |
80 | 81 | 82 | -------------------------------------------------------------------------------- /concorrencia/async/mojifinder/tcp_mojifinder.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # tag::TCP_MOJIFINDER_TOP[] 4 | import asyncio 5 | import sys 6 | from asyncio.trsock import TransportSocket 7 | from typing import cast 8 | 9 | from charindex import InvertedIndex, format_results # <1> 10 | 11 | CRLF = b'\r\n' 12 | PROMPT = b'?> ' 13 | 14 | index = None 15 | 16 | async def finder(reader: asyncio.StreamReader, 17 | writer: asyncio.StreamWriter) -> None: 18 | client = writer.get_extra_info('peername') # <3> 19 | while True: # <4> 20 | writer.write(PROMPT) # can't await! # <5> 21 | await writer.drain() # must await! # <6> 22 | data = await reader.readline() # <7> 23 | if not data: # <8> 24 | break 25 | try: 26 | query = data.decode().strip() # <9> 27 | except UnicodeDecodeError: # <10> 28 | query = '\x00' 29 | print(f' From {client}: {query!r}') # <11> 30 | if query: 31 | if ord(query[:1]) < 32: # <12> 32 | break 33 | results = await search(query, writer) # <13> 34 | print(f' To {client}: {results} results.') # <14> 35 | 36 | writer.close() # <15> 37 | await writer.wait_closed() # <16> 38 | print(f'Close {client}.') # <17> 39 | # end::TCP_MOJIFINDER_TOP[] 40 | 41 | # tag::TCP_MOJIFINDER_SEARCH[] 42 | async def search(query: str, # <1> 43 | writer: asyncio.StreamWriter) -> int: 44 | chars = index.search(query) # <2> 45 | lines = (line.encode() + CRLF for line # <3> 46 | in format_results(chars)) 47 | writer.writelines(lines) # <4> 48 | await writer.drain() # <5> 49 | status_line = f'{"─" * 66} {len(chars)} found' # <6> 50 | writer.write(status_line.encode() + CRLF) 51 | await writer.drain() 52 | return len(chars) 53 | # end::TCP_MOJIFINDER_SEARCH[] 54 | 55 | # tag::TCP_MOJIFINDER_MAIN[] 56 | async def supervisor(host: str, port: int) -> None: 57 | server = await asyncio.start_server(finder, host, port) 58 | 59 | socket_list = cast(tuple[TransportSocket, ...], server.sockets) # <4> 60 | addr = socket_list[0].getsockname() 61 | print(f'Serving on {addr}. Hit CTRL-C to stop.') # <5> 62 | await server.serve_forever() # <6> 63 | 64 | def main(host: str = '127.0.0.1', port_arg: str = '2323'): 65 | global index 66 | port = int(port_arg) 67 | print('Building index.') 68 | index = InvertedIndex() # <7> 69 | try: 70 | asyncio.run(supervisor(host, port)) # <8> 71 | except KeyboardInterrupt: # <9> 72 | print('\nServer shut down.') 73 | 74 | if __name__ == '__main__': 75 | main(*sys.argv[1:]) 76 | # end::TCP_MOJIFINDER_MAIN[] 77 | -------------------------------------------------------------------------------- /concorrencia/async/mojifinder/web_mojifinder.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from unicodedata import name 3 | 4 | from fastapi import FastAPI 5 | from fastapi.responses import HTMLResponse 6 | from pydantic import BaseModel 7 | 8 | from charindex import InvertedIndex 9 | 10 | STATIC_PATH = Path(__file__).parent.absolute() / 'static' # <1> 11 | 12 | app = FastAPI( # <2> 13 | title='Mojifinder Web', 14 | description='Search for Unicode characters by name.', 15 | ) 16 | 17 | class CharName(BaseModel): # <3> 18 | char: str 19 | name: str 20 | 21 | def init(app): # <4> 22 | app.state.index = InvertedIndex() 23 | app.state.form = (STATIC_PATH / 'form.html').read_text() 24 | 25 | init(app) # <5> 26 | 27 | @app.get('/search', response_model=list[CharName]) # <6> 28 | async def search(q: str): # <7> 29 | # nesse bloco eu posso usar await!!!! 30 | chars = sorted(app.state.index.search(q)) 31 | return (CharName(char=c, name=name(c)) for c in chars) # <8> 32 | 33 | @app.get('/', response_class=HTMLResponse, include_in_schema=False) 34 | def form(): # <9> 35 | return app.state.form 36 | 37 | # no main funcion # <10> -------------------------------------------------------------------------------- /concorrencia/async/mojifinder/web_mojifinder.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Installation: 3 | # $ pip install fastapi uvicorn 4 | uvicorn web_mojifinder:app 5 | -------------------------------------------------------------------------------- /concorrencia/async/mojifinder/web_mojifinder_bottle.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import unicodedata 5 | 6 | from bottle import route, request, run, static_file 7 | 8 | from charindex import InvertedIndex 9 | 10 | index = {} 11 | 12 | @route('/') 13 | def form(): 14 | return static_file('form.html', root='static/') 15 | 16 | 17 | @route('/search') 18 | def search(): 19 | query = request.query['q'] 20 | chars = index.search(query) 21 | results = [] 22 | for char in chars: 23 | name = unicodedata.name(char) 24 | results.append({'char': char, 'name': name}) 25 | return json.dumps(results).encode('UTF-8') 26 | 27 | 28 | def main(port): 29 | global index 30 | index = InvertedIndex() 31 | run(host='localhost', port=port, debug=True) 32 | 33 | 34 | if __name__ == '__main__': 35 | main(8000) 36 | -------------------------------------------------------------------------------- /concorrencia/clock.py: -------------------------------------------------------------------------------- 1 | from time import sleep 2 | from datetime import datetime 3 | 4 | 5 | while True: 6 | now = datetime.now().strftime('%H:%M:%S.%f')[:10] 7 | try: 8 | print(now, end='', flush=True) 9 | sleep(.1) 10 | except KeyboardInterrupt: 11 | break 12 | finally: 13 | print('\r' * len(now), end='', flush=True) 14 | -------------------------------------------------------------------------------- /concorrencia/clock_face.py: -------------------------------------------------------------------------------- 1 | from time import sleep 2 | from unicodedata import lookup 3 | 4 | hours = 'ONE TWO THREE FOUR FIVE SIX SEVEN EIGHT NINE TEN ELEVEN TWELVE'.split() 5 | 6 | faces = [lookup(f'CLOCK FACE {h} OCLOCK') for h in hours] 7 | 8 | face = 0 9 | while True: 10 | now = faces[face] 11 | face = (face + 1) % len(faces) 12 | print(now, end='', flush=True) 13 | sleep(.1) 14 | print('\r' * len(now), end='', flush=True) 15 | -------------------------------------------------------------------------------- /concorrencia/concorrencia-x-paralelismo.odp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramalho/python-eng/59a007d54595f11441ec7de07909576e1c158772/concorrencia/concorrencia-x-paralelismo.odp -------------------------------------------------------------------------------- /concorrencia/concorrencia-x-paralelismo.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramalho/python-eng/59a007d54595f11441ec7de07909576e1c158772/concorrencia/concorrencia-x-paralelismo.pdf -------------------------------------------------------------------------------- /concorrencia/curio-example/curio-example.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "bf4f7db9-ab8f-40c6-8890-8c85fca5503b", 6 | "metadata": {}, 7 | "source": [ 8 | "# Exemplo com a biblioteca curio\n", 9 | "\n", 10 | "**[curio](https://github.com/dabeaz/curio/)** é uma biblioteca de programação assíncrona experimental do professor David Beazley que apresentou novas formas de usar as instruções `async for` e `async wait`, de uma forma mais *pythonica* do que a própria biblioteca `asyncio` (que foi criada antes das instruções `async`).\n", 11 | "\n", 12 | "Este exemplo funciona, mas só usando a versão mais recente do\n", 13 | "[repositório do curio](https://github.com/dabeaz/curio/) e não o pacote `curio` do PyPI." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 1, 19 | "id": "890fb706-241c-45f7-8213-166f43d5467e", 20 | "metadata": {}, 21 | "outputs": [ 22 | { 23 | "name": "stdout", 24 | "output_type": "stream", 25 | "text": [ 26 | " none.dev\n", 27 | " for.dev\n", 28 | " or.dev\n", 29 | "✓ false.dev\n", 30 | "✓ and.dev\n", 31 | "✓ as.dev\n", 32 | "✓ while.dev\n", 33 | "✓ in.dev\n", 34 | "✓ try.dev\n", 35 | "✓ await.dev\n", 36 | "✓ def.dev\n", 37 | " if.dev\n", 38 | "✓ not.dev\n", 39 | " elif.dev\n", 40 | " true.dev\n", 41 | " pass.dev\n", 42 | " is.dev\n", 43 | " class.dev\n", 44 | "✓ from.dev\n", 45 | " yield.dev\n", 46 | " break.dev\n", 47 | " else.dev\n", 48 | " with.dev\n", 49 | "✓ async.dev\n", 50 | "✓ del.dev\n", 51 | "✓ raise.dev\n", 52 | "26 domains probed in 0.5212s\n" 53 | ] 54 | } 55 | ], 56 | "source": [ 57 | "from curio import run, TaskGroup\n", 58 | "import curio.socket as socket\n", 59 | "from keyword import kwlist\n", 60 | "\n", 61 | "MAX_KEYWORD_LEN = 5\n", 62 | "NAMES = [kw for kw in kwlist if len(kw) <= MAX_KEYWORD_LEN]\n", 63 | "\n", 64 | "async def probe(domain: str) -> tuple[str, bool]: # (1)\n", 65 | " try:\n", 66 | " await socket.getaddrinfo(domain, None) # (2)\n", 67 | " except socket.gaierror:\n", 68 | " return (domain, False)\n", 69 | " return (domain, True)\n", 70 | "\n", 71 | "async def main() -> None:\n", 72 | " domains = (f'{name}.dev'.lower() for name in NAMES)\n", 73 | " async with TaskGroup() as group: # (3)\n", 74 | " for domain in domains:\n", 75 | " await group.spawn(probe, domain) # (4)\n", 76 | " async for task in group: # (5)\n", 77 | " domain, found = task.result\n", 78 | " mark = '✓' if found else ' '\n", 79 | " print(f'{mark} {domain}')\n", 80 | "\n", 81 | "if __name__ == '__main__':\n", 82 | " from time import perf_counter\n", 83 | " t0 = perf_counter()\n", 84 | " run(main()) # (6)\n", 85 | " dt = perf_counter() - t0\n", 86 | " print(f'{len(NAMES)} domains probed in {dt:0.4f}s')" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": null, 92 | "id": "7a0243ec-1f56-44fe-a587-b285da75bb15", 93 | "metadata": {}, 94 | "outputs": [], 95 | "source": [] 96 | } 97 | ], 98 | "metadata": { 99 | "kernelspec": { 100 | "display_name": "Python 3 (ipykernel)", 101 | "language": "python", 102 | "name": "python3" 103 | }, 104 | "language_info": { 105 | "codemirror_mode": { 106 | "name": "ipython", 107 | "version": 3 108 | }, 109 | "file_extension": ".py", 110 | "mimetype": "text/x-python", 111 | "name": "python", 112 | "nbconvert_exporter": "python", 113 | "pygments_lexer": "ipython3", 114 | "version": "3.12.2" 115 | } 116 | }, 117 | "nbformat": 4, 118 | "nbformat_minor": 5 119 | } 120 | -------------------------------------------------------------------------------- /concorrencia/curio-example/curio/README.md: -------------------------------------------------------------------------------- 1 | # curio 2 | 3 | Vendored copy of David Beazley's `curio` library from the 4 | [official repository](https://github.com/dabeaz/curio) 5 | as of 2024-04-11, commit hash `e85889e910c6cc56d8dbcef9870cb8a6101076bc`. 6 | 7 | The [curio PyPI package](https://pypi.org/project/curio/) 8 | is not compatible with Python 3.12 and is no longer updated. 9 | -------------------------------------------------------------------------------- /concorrencia/curio-example/curio/__init__.py: -------------------------------------------------------------------------------- 1 | # curio/__init__.py 2 | 3 | __version__ = '1.6' 4 | 5 | from .errors import * 6 | from .queue import * 7 | from .task import * 8 | from .time import * 9 | from .kernel import * 10 | from .sync import * 11 | from .workers import * 12 | from .network import * 13 | from .file import * 14 | from .channel import * 15 | from .thread import * 16 | 17 | __all__ = [*errors.__all__, 18 | *queue.__all__, 19 | *task.__all__, 20 | *time.__all__, 21 | *kernel.__all__, 22 | *sync.__all__, 23 | *workers.__all__, 24 | *network.__all__, 25 | *file.__all__, 26 | *channel.__all__, 27 | *thread.__all__, 28 | ] 29 | -------------------------------------------------------------------------------- /concorrencia/curio-example/curio/__main__.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import curio 3 | import curio.monitor 4 | import code 5 | import inspect 6 | import sys 7 | import types 8 | import warnings 9 | import threading 10 | import signal 11 | import os 12 | 13 | assert (sys.version_info.major >= 3 and sys.version_info.minor >= 8), "console requires Python 3.8+" 14 | 15 | class CurioIOInteractiveConsole(code.InteractiveConsole): 16 | 17 | def __init__(self, locals): 18 | super().__init__(locals) 19 | self.compile.compiler.flags |= ast.PyCF_ALLOW_TOP_LEVEL_AWAIT 20 | self.requests = curio.UniversalQueue() 21 | self.response = curio.UniversalQueue() 22 | 23 | def runcode(self, code): 24 | # This coroutine is handed from the thread running the REPL to the 25 | # task runner in the main thread. 26 | async def run_it(): 27 | func = types.FunctionType(code, self.locals) 28 | try: 29 | # We restore the default REPL signal handler for running normal code 30 | hand = signal.signal(signal.SIGINT, signal.default_int_handler) 31 | try: 32 | coro = func() 33 | finally: 34 | signal.signal(signal.SIGINT, hand) 35 | except BaseException as ex: 36 | await self.response.put((None, ex)) 37 | return 38 | if not inspect.iscoroutine(coro): 39 | await self.response.put((coro, None)) 40 | return 41 | 42 | # For a coroutine... We're going to try and do some magic to intercept 43 | # Control-C in an Event/Task. 44 | async def watch_ctrl_c(evt, repl_task): 45 | await evt.wait() 46 | await repl_task.cancel() 47 | evt = curio.UniversalEvent() 48 | try: 49 | hand = signal.signal(signal.SIGINT, lambda signo, frame: evt.set()) 50 | repl_task = await curio.spawn(coro) 51 | watch_task = await curio.spawn(watch_ctrl_c, evt, repl_task) 52 | try: 53 | result = await repl_task.join() 54 | response = (result, None) 55 | except SystemExit: 56 | raise 57 | except BaseException as e: 58 | await repl_task.wait() 59 | response = (None, e.__cause__) 60 | await watch_task.cancel() 61 | finally: 62 | signal.signal(signal.SIGINT, hand) 63 | await self.response.put(response) 64 | 65 | self.requests.put(run_it()) 66 | # Get the result here... 67 | result, exc = self.response.get() 68 | if exc is not None: 69 | try: 70 | raise exc 71 | except BaseException: 72 | self.showtraceback() 73 | else: 74 | return result 75 | 76 | # Task that runs in the main thread, executing input fed to it from above 77 | async def runmain(self): 78 | try: 79 | hand = signal.signal(signal.SIGINT, signal.SIG_IGN) 80 | while True: 81 | coro = await self.requests.get() 82 | if coro is None: 83 | break 84 | await coro 85 | finally: 86 | signal.signal(signal.SIGINT, hand) 87 | 88 | def run_repl(console): 89 | try: 90 | banner = ( 91 | f'curio REPL {sys.version} on {sys.platform}\n' 92 | f'Use "await" directly instead of "curio.run()".\n' 93 | f'Type "help", "copyright", "credits" or "license" ' 94 | f'for more information.\n' 95 | f'{getattr(sys, "ps1", ">>> ")}import curio' 96 | ) 97 | console.interact( 98 | banner=banner, 99 | exitmsg='exiting curio REPL...') 100 | finally: 101 | warnings.filterwarnings( 102 | 'ignore', 103 | message=r'^coroutine .* was never awaited$', 104 | category=RuntimeWarning) 105 | console.requests.put(None) 106 | 107 | if __name__ == '__main__': 108 | repl_locals = { 'curio': curio, 109 | 'ps': curio.monitor.ps, 110 | 'where': curio.monitor.where, 111 | } 112 | for key in {'__name__', '__package__', 113 | '__loader__', '__spec__', 114 | '__builtins__', '__file__'}: 115 | repl_locals[key] = locals()[key] 116 | 117 | console = CurioIOInteractiveConsole(repl_locals) 118 | threading.Thread(target=run_repl, args=[console], daemon=True).start() 119 | curio.run(console.runmain) 120 | -------------------------------------------------------------------------------- /concorrencia/curio-example/curio/debug.py: -------------------------------------------------------------------------------- 1 | # curio/debug.py 2 | # 3 | # Task debugging tools 4 | 5 | __all__ = [ 'longblock', 'schedtrace', 'traptrace', 'logcrash' ] 6 | 7 | import time 8 | import logging 9 | log = logging.getLogger(__name__) 10 | 11 | # -- Curio 12 | 13 | from .kernel import Activation 14 | from .errors import TaskCancelled 15 | 16 | class DebugBase(Activation): 17 | def __init__(self, *, level=logging.INFO, log=log, filter=None, **kwargs): 18 | self.level = level 19 | self.filter = filter 20 | self.log = log 21 | 22 | def check_filter(self, task): 23 | if self.filter and task.name not in self.filter: 24 | return False 25 | return True 26 | 27 | class longblock(DebugBase): 28 | ''' 29 | Report warnings for tasks that block the event loop for a long duration. 30 | ''' 31 | def __init__(self, *, max_time=0.05, level=logging.WARNING, **kwargs): 32 | super().__init__(level=level, **kwargs) 33 | self.max_time = max_time 34 | 35 | def running(self, task): 36 | if self.check_filter(task): 37 | self.start = time.monotonic() 38 | 39 | def suspended(self, task, trap): 40 | if self.check_filter(task): 41 | duration = time.monotonic() - self.start 42 | if duration > self.max_time: 43 | self.log.log(self.level, '%r ran for %s seconds', task, duration) 44 | 45 | class logcrash(DebugBase): 46 | ''' 47 | Report tasks that crash with an uncaught exception 48 | ''' 49 | def __init__(self, level=logging.ERROR, **kwargs): 50 | super().__init__(level=level, **kwargs) 51 | 52 | def suspended(self, task, trap): 53 | if task.terminated and self.check_filter(task): 54 | if task.exception and not isinstance(task.exception, (StopIteration, TaskCancelled, KeyboardInterrupt, SystemExit)): 55 | self.log.log(self.level, '%r crashed', task, exc_info=task.exception) 56 | 57 | class schedtrace(DebugBase): 58 | ''' 59 | Report when tasks run 60 | ''' 61 | def __init__(self, **kwargs): 62 | super().__init__(**kwargs) 63 | 64 | def created(self, task): 65 | if self.check_filter(task): 66 | self.log.log(self.level, 'CREATE:%f:%r', time.time(), task) 67 | 68 | def running(self, task): 69 | if self.check_filter(task): 70 | self.log.log(self.level, 'RUN:%f:%r', time.time(), task) 71 | 72 | def suspended(self, task, trap): 73 | if self.check_filter(task): 74 | self.log.log(self.level, 'SUSPEND:%f:%r', time.time(), task) 75 | 76 | def terminated(self, task): 77 | if self.check_filter(task): 78 | self.log.log(self.level, 'TERMINATED:%f:%r', time.time(), task) 79 | 80 | class traptrace(schedtrace): 81 | ''' 82 | Report traps executed 83 | ''' 84 | def suspended(self, task, trap): 85 | if self.check_filter(task): 86 | if trap: 87 | self.log.log(self.level, 'TRAP:%r', trap) 88 | super().suspended(task, trap) 89 | 90 | def _create_debuggers(debug): 91 | ''' 92 | Create debugger objects. Called by the kernel to instantiate the objects. 93 | ''' 94 | if debug is True: 95 | # Set a default set of debuggers 96 | debug = [ schedtrace ] 97 | 98 | elif not isinstance(debug, (list, tuple)): 99 | debug = [ debug ] 100 | 101 | # Create instances 102 | debug = [ (d() if (isinstance(d, type) and issubclass(d, DebugBase)) else d) 103 | for d in debug ] 104 | return debug 105 | 106 | 107 | 108 | 109 | 110 | 111 | -------------------------------------------------------------------------------- /concorrencia/curio-example/curio/errors.py: -------------------------------------------------------------------------------- 1 | # curio/errors.py 2 | # 3 | # Curio specific exceptions 4 | 5 | __all__ = [ 6 | 'CurioError', 'CancelledError', 'TaskTimeout', 'TaskError', 7 | 'SyncIOError', 'ResourceBusy', 8 | 'ReadResourceBusy', 'WriteResourceBusy', 9 | 'TimeoutCancellationError', 'UncaughtTimeoutError', 10 | 'TaskCancelled', 'AsyncOnlyError', 11 | ] 12 | 13 | 14 | class CurioError(Exception): 15 | ''' 16 | Base class for all non-cancellation Curio-related exceptions 17 | ''' 18 | 19 | 20 | class CancelledError(BaseException): 21 | ''' 22 | Base class for all task-cancellation related exceptions 23 | ''' 24 | 25 | 26 | class TaskCancelled(CancelledError): 27 | ''' 28 | Exception raised as a result of a task being directly cancelled. 29 | ''' 30 | 31 | 32 | class TimeoutCancellationError(CancelledError): 33 | ''' 34 | Exception raised if task is being cancelled due to a timeout, but 35 | it's not the inner-most timeout in effect. 36 | ''' 37 | 38 | 39 | class TaskTimeout(CancelledError): 40 | ''' 41 | Exception raised if task is cancelled due to timeout. 42 | ''' 43 | 44 | 45 | class UncaughtTimeoutError(CurioError): 46 | ''' 47 | Raised if a TaskTimeout exception escapes a timeout handling 48 | block and is unexpectedly caught by an outer timeout handler. 49 | ''' 50 | 51 | 52 | class TaskError(CurioError): 53 | ''' 54 | Raised if a task launched via spawn() or similar function 55 | terminated due to an exception. This is a chained exception. 56 | The __cause__ attribute contains the actual exception that 57 | occurred in the task. 58 | ''' 59 | 60 | 61 | class SyncIOError(CurioError): 62 | ''' 63 | Raised if a task attempts to perform a synchronous I/O operation 64 | on an object that only supports asynchronous I/O. 65 | ''' 66 | 67 | 68 | class AsyncOnlyError(CurioError): 69 | ''' 70 | Raised by the AWAIT() function if its applied to code not 71 | properly running in an async-thread. 72 | ''' 73 | 74 | 75 | class ResourceBusy(CurioError): 76 | ''' 77 | Raised by I/O related functions if an operation is requested, 78 | but the resource is already busy performing the same operation 79 | on behalf of another task. 80 | ''' 81 | 82 | 83 | class ReadResourceBusy(ResourceBusy): 84 | pass 85 | 86 | 87 | class WriteResourceBusy(ResourceBusy): 88 | pass 89 | 90 | 91 | -------------------------------------------------------------------------------- /concorrencia/curio-example/curio/file.py: -------------------------------------------------------------------------------- 1 | # curio/file.py 2 | # 3 | # Let's talk about files for a moment. Suppose you're in a coroutine 4 | # and you start using things like the built-in open() function: 5 | # 6 | # async def coro(): 7 | # f = open(somefile, 'r') 8 | # data = f.read() 9 | # ... 10 | # 11 | # Yes, it will "work", but who knows what's actually going to happen 12 | # on that open() call and associated read(). If it's on disk, the 13 | # whole program might lock up for a few milliseconds (aka. "an 14 | # eternity") doing a disk seek. While that happens, your whole 15 | # coroutine based server is going to grind to a screeching halt. This 16 | # is bad--especially if a lot of coroutines start doing it all at 17 | # once. 18 | # 19 | # Knowing how to handle this is a tricky question. Traditional files 20 | # don't really support "async" in the usual way a socket might. You 21 | # might be able to do something sneaky with asynchronous POSIX APIs 22 | # (i.e., aio_* functions) or maybe thread pools. However, one thing 23 | # is for certain--if files are going to be handled in a sane way, they're 24 | # going to have an async interface. 25 | # 26 | # This file does just that by providing an async-compatible aopen() 27 | # call. You use it the same way you use open() and a normal file: 28 | # 29 | # async def coro(): 30 | # async with aopen(somefile, 'r') as f: 31 | # data = await f.read() 32 | # ... 33 | # 34 | # If you want to use iteration, make sure you use the asynchronous version: 35 | # 36 | # async def coro(): 37 | # async with aopen(somefile, 'r') as f: 38 | # async for line in f: 39 | # ... 40 | # 41 | 42 | __all__ = ['aopen', 'anext'] 43 | 44 | # -- Standard library 45 | 46 | from contextlib import contextmanager 47 | from functools import partial 48 | 49 | # -- Curio 50 | 51 | from .workers import run_in_thread 52 | from .errors import SyncIOError, CancelledError 53 | from . import thread 54 | 55 | class AsyncFile(object): 56 | ''' 57 | An async wrapper around a standard file object. Uses threads to 58 | execute various I/O operations in a way that avoids blocking 59 | the Curio kernel loop. 60 | ''' 61 | 62 | def __init__(self, fileobj, open_args=None, open_kwargs=None): 63 | self._fileobj = fileobj 64 | self._open_args = open_args 65 | self._open_kwargs = open_kwargs 66 | 67 | def __repr__(self): 68 | return 'AsyncFile(%r)' % self._fileobj 69 | 70 | @contextmanager 71 | def blocking(self): 72 | ''' 73 | Expose the underlying file in blocking mode for use with synchronous code. 74 | ''' 75 | yield self._file 76 | 77 | @property 78 | def _file(self): 79 | if self._fileobj is None: 80 | raise RuntimeError('Must use an async file as an async-context-manager.') 81 | return self._fileobj 82 | 83 | async def read(self, *args, **kwargs): 84 | return await run_in_thread(partial(self._file.read, *args, **kwargs)) 85 | 86 | async def read1(self, *args, **kwargs): 87 | return await run_in_thread(partial(self._file.read1, *args, **kwargs)) 88 | 89 | async def readinto(self, *args, **kwargs): 90 | return await run_in_thread(partial(self._file.readinto, *args, **kwargs)) 91 | 92 | async def readinto1(self, *args, **kwargs): 93 | return await run_in_thread(partial(self._file.readinto1, *args, **kwargs)) 94 | 95 | async def readline(self, *args, **kwargs): 96 | return await run_in_thread(partial(self._file.readline, *args, **kwargs)) 97 | 98 | async def readlines(self, *args, **kwargs): 99 | return await run_in_thread(partial(self._file.readlines, *args, **kwargs)) 100 | 101 | async def write(self, *args, **kwargs): 102 | return await run_in_thread(partial(self._file.write, *args, **kwargs)) 103 | 104 | async def writelines(self, *args, **kwargs): 105 | return await run_in_thread(partial(self._file.writelines, *args, **kwargs)) 106 | 107 | async def flush(self): 108 | return await run_in_thread(self._file.flush) 109 | 110 | async def close(self): 111 | return await run_in_thread(self._file.close) 112 | 113 | async def seek(self, *args, **kwargs): 114 | return await run_in_thread(partial(self._file.seek, *args, **kwargs)) 115 | 116 | async def tell(self, *args, **kwargs): 117 | return await run_in_thread(partial(self._file.tell, *args, **kwargs)) 118 | 119 | async def truncate(self, *args, **kwargs): 120 | return await run_in_thread(partial(self._file.truncate, *args, **kwargs)) 121 | 122 | def __iter__(self): 123 | raise SyncIOError('Use asynchronous iteration') 124 | 125 | def __next__(self): 126 | raise SyncIOError('Use asynchronous iteration') 127 | 128 | def __enter__(self): 129 | return thread.AWAIT(self.__aenter__()) 130 | 131 | def __exit__(self, *args): 132 | return thread.AWAIT(self.__aexit__(*args)) 133 | 134 | def __aiter__(self): 135 | return self 136 | 137 | async def __aenter__(self): 138 | if self._fileobj is None: 139 | self._fileobj = await run_in_thread(partial(open, *self._open_args, **self._open_kwargs)) 140 | return self 141 | 142 | async def __aexit__(self, *args): 143 | await self.close() 144 | 145 | async def __anext__(self): 146 | data = await run_in_thread(next, self._file, None) 147 | if data is None: 148 | raise StopAsyncIteration 149 | return data 150 | 151 | def __getattr__(self, name): 152 | return getattr(self._file, name) 153 | 154 | # Compatibility with io.FileStream 155 | async def readall(self): 156 | chunks = [] 157 | maxread = 65536 158 | sep = '' if hasattr(self._file, 'encoding') else b'' 159 | while True: 160 | try: 161 | chunk = await self.read(maxread) 162 | except CancelledError as e: 163 | e.bytes_read = sep.join(chunks) 164 | raise 165 | if not chunk: 166 | return sep.join(chunks) 167 | chunks.append(chunk) 168 | if len(chunk) == maxread: 169 | maxread *= 2 170 | 171 | def aopen(*args, **kwargs): 172 | ''' 173 | Async version of the builtin open() function that returns an async-compatible 174 | file object. Takes the same arguments. Returns a wrapped file in which 175 | blocking I/O operations must be awaited. 176 | ''' 177 | return AsyncFile(None, args, kwargs) 178 | 179 | async def anext(f, sentinel=object): 180 | ''' 181 | Async version of the builtin next() function that advances an async iterator. 182 | Sometimes used to skip a single line in files. 183 | ''' 184 | try: 185 | return await f.__anext__() 186 | except StopAsyncIteration: 187 | if sentinel is not object: 188 | return sentinel 189 | else: 190 | raise 191 | -------------------------------------------------------------------------------- /concorrencia/curio-example/curio/network.py: -------------------------------------------------------------------------------- 1 | # curio/network.py 2 | # 3 | # Some high-level functions useful for writing network code. These are loosely 4 | # based on their similar counterparts in the asyncio library. Some of the 5 | # fiddly low-level bits are borrowed. 6 | 7 | __all__ = [ 'open_connection', 'tcp_server', 'tcp_server_socket', 8 | 'open_unix_connection', 'unix_server', 'unix_server_socket' ] 9 | 10 | # -- Standard library 11 | 12 | import logging 13 | log = logging.getLogger(__name__) 14 | 15 | # -- Curio 16 | 17 | from . import socket 18 | from . import ssl as curiossl 19 | from .task import TaskGroup 20 | from .io import Socket 21 | 22 | 23 | async def _wrap_ssl_client(sock, ssl, server_hostname, alpn_protocols): 24 | # Applies SSL to a client connection. Returns an SSL socket. 25 | if ssl: 26 | if isinstance(ssl, bool): 27 | sslcontext = curiossl.create_default_context() 28 | if not server_hostname: 29 | sslcontext._context.check_hostname = False 30 | sslcontext._context.verify_mode = curiossl.CERT_NONE 31 | 32 | if alpn_protocols: 33 | sslcontext.set_alpn_protocols(alpn_protocols) 34 | else: 35 | # Assume that ssl is an already created context 36 | sslcontext = ssl 37 | 38 | if server_hostname: 39 | extra_args = {'server_hostname': server_hostname} 40 | else: 41 | extra_args = {} 42 | 43 | # if the context is Curio's own, it expects a Curio socket and 44 | # returns one. If context is from an external source, including 45 | # the stdlib's ssl.SSLContext, it expects a non-Curio socket and 46 | # returns a non-Curio socket, which then needs wrapping in a Curio 47 | # socket. 48 | # 49 | # Perhaps the CurioSSLContext is no longer needed. In which case, 50 | # this code can be simplified to just the else case below. 51 | # 52 | if isinstance(sslcontext, curiossl.CurioSSLContext): 53 | sock = await sslcontext.wrap_socket(sock, do_handshake_on_connect=False, **extra_args) 54 | else: 55 | # do_handshake_on_connect should not be specified for 56 | # non-blocking sockets 57 | extra_args['do_handshake_on_connect'] = sock._socket.gettimeout() != 0.0 58 | sock = Socket(sslcontext.wrap_socket(sock._socket, **extra_args)) 59 | await sock.do_handshake() 60 | return sock 61 | 62 | async def open_connection(host, port, *, ssl=None, source_addr=None, server_hostname=None, 63 | alpn_protocols=None): 64 | ''' 65 | Create a TCP connection to a given Internet host and port with optional SSL applied to it. 66 | ''' 67 | if server_hostname and not ssl: 68 | raise ValueError('server_hostname is only applicable with SSL') 69 | 70 | sock = await socket.create_connection((host, port), source_address=source_addr) 71 | 72 | try: 73 | # Apply SSL wrapping to the connection, if applicable 74 | if ssl: 75 | sock = await _wrap_ssl_client(sock, ssl, server_hostname, alpn_protocols) 76 | 77 | return sock 78 | except Exception: 79 | sock._socket.close() 80 | raise 81 | 82 | async def open_unix_connection(path, *, ssl=None, server_hostname=None, 83 | alpn_protocols=None): 84 | if server_hostname and not ssl: 85 | raise ValueError('server_hostname is only applicable with SSL') 86 | 87 | sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 88 | try: 89 | await sock.connect(path) 90 | 91 | # Apply SSL wrapping to connection, if applicable 92 | if ssl: 93 | sock = await _wrap_ssl_client(sock, ssl, server_hostname, alpn_protocols) 94 | 95 | return sock 96 | except Exception: 97 | sock._socket.close() 98 | raise 99 | 100 | async def run_server(sock, client_connected_task, ssl=None): 101 | if ssl and not hasattr(ssl, 'wrap_socket'): 102 | raise ValueError('ssl argument must have a wrap_socket method') 103 | 104 | async def run_client(client, addr): 105 | async with client: 106 | await client_connected_task(client, addr) 107 | 108 | async def run_server(sock, group): 109 | while True: 110 | client, addr = await sock.accept() 111 | if ssl: 112 | if isinstance(ssl, curiossl.CurioSSLContext): 113 | client = await ssl.wrap_socket(client, server_side=True, do_handshake_on_connect=False) 114 | else: 115 | client = ssl.wrap_socket(client, server_side=True, do_handshake_on_connect=False) 116 | if not isinstance(client, Socket): 117 | client = Socket(client) 118 | await group.spawn(run_client, client, addr) 119 | del client 120 | 121 | async with sock: 122 | async with TaskGroup() as tg: 123 | await tg.spawn(run_server, sock, tg) 124 | # Reap all of the children tasks as they complete 125 | async for task in tg: 126 | task.joined = True 127 | del task 128 | 129 | def tcp_server_socket(host, port, family=socket.AF_INET, backlog=100, 130 | reuse_address=True, reuse_port=False): 131 | 132 | sock = socket.socket(family, socket.SOCK_STREAM) 133 | try: 134 | if reuse_address: 135 | sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) 136 | 137 | if reuse_port: 138 | try: 139 | sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, True) 140 | except (AttributeError, OSError) as e: 141 | log.warning('reuse_port=True option failed', exc_info=True) 142 | 143 | sock.bind((host, port)) 144 | sock.listen(backlog) 145 | except Exception: 146 | sock._socket.close() 147 | raise 148 | 149 | return sock 150 | 151 | async def tcp_server(host, port, client_connected_task, *, 152 | family=socket.AF_INET, backlog=100, ssl=None, 153 | reuse_address=True, reuse_port=False): 154 | 155 | sock = tcp_server_socket(host, port, family, backlog, reuse_address, reuse_port) 156 | await run_server(sock, client_connected_task, ssl) 157 | 158 | def unix_server_socket(path, backlog=100): 159 | sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 160 | try: 161 | sock.bind(path) 162 | sock.listen(backlog) 163 | except Exception: 164 | sock._socket.close() 165 | raise 166 | return sock 167 | 168 | async def unix_server(path, client_connected_task, *, backlog=100, ssl=None): 169 | sock = unix_server_socket(path, backlog) 170 | await run_server(sock, client_connected_task, ssl) 171 | -------------------------------------------------------------------------------- /concorrencia/curio-example/curio/sched.py: -------------------------------------------------------------------------------- 1 | # curio/sched.py 2 | # 3 | # Task-scheduling primitives. These are used to implement low-level 4 | # scheduling operations needed by higher-level abstractions such 5 | # as Events, Locks, Semaphores, and Queues. 6 | 7 | __all__ = [ 'SchedFIFO', 'SchedBarrier' ] 8 | 9 | # -- Standard Library 10 | 11 | from abc import ABC, abstractmethod 12 | from collections import deque 13 | 14 | # -- Curio 15 | 16 | from .traps import _scheduler_wait, _scheduler_wake 17 | 18 | 19 | class SchedBase(ABC): 20 | 21 | def __repr__(self): 22 | return f'{type(self).__name__}<{len(self)} tasks waiting>' 23 | 24 | @abstractmethod 25 | def __len__(self): 26 | pass 27 | 28 | @abstractmethod 29 | def _kernel_suspend(self, task): 30 | ''' 31 | Suspends a task. This method *must* return a zero-argument 32 | callable that removes the just added task from the scheduler 33 | on cancellation. Called by the kernel. 34 | ''' 35 | pass 36 | 37 | @abstractmethod 38 | def _kernel_wake(self, ntasks=1): 39 | ''' 40 | Wake one or more tasks. Returns a list of the awakened tasks. 41 | Called by the kernel. 42 | ''' 43 | pass 44 | 45 | async def suspend(self, reason='SUSPEND'): 46 | ''' 47 | Suspend the calling task. reason is a string containing 48 | descriptive text to indicate why (used to set the task state). 49 | ''' 50 | await _scheduler_wait(self, reason) 51 | 52 | async def wake(self, n=1): 53 | ''' 54 | Wake one or more suspended tasks. 55 | ''' 56 | await _scheduler_wake(self, n) 57 | 58 | 59 | class SchedFIFO(SchedBase): 60 | ''' 61 | A scheduling FIFO. Tasks sleep and awake in the order of arrival. 62 | The wake method only awakens a single task. Commonly used to 63 | implement locks and queues. 64 | ''' 65 | def __init__(self): 66 | self._queue = deque() 67 | self._actual_len = 0 68 | 69 | def __len__(self): 70 | return self._actual_len 71 | 72 | def _kernel_suspend(self, task): 73 | # The task is placed inside a 1-item list. If cancelled, the 74 | # task is replaced by None, but the list remains on the queue 75 | # until later pop operations discard it 76 | item = [task] 77 | self._queue.append(item) 78 | self._actual_len += 1 79 | 80 | def remove(): 81 | item[0] = None 82 | self._actual_len -= 1 83 | return remove 84 | 85 | def _kernel_wake(self, ntasks=1): 86 | tasks = [] 87 | while ntasks > 0: 88 | task, = self._queue.popleft() 89 | if task: 90 | tasks.append(task) 91 | ntasks -= 1 92 | self._actual_len -= len(tasks) 93 | return tasks 94 | 95 | class SchedBarrier(SchedBase): 96 | ''' 97 | A scheduling barrier. Sleeping tasks are collected into a set. 98 | Waking makes all of the blocked tasks reawaken at the same time. 99 | Commonly used to implement Event and join(). 100 | ''' 101 | def __init__(self): 102 | self._tasks = set() 103 | 104 | def __len__(self): 105 | return len(self._tasks) 106 | 107 | def _kernel_suspend(self, task): 108 | self._tasks.add(task) 109 | return lambda: self._tasks.remove(task) 110 | 111 | def _kernel_wake(self, ntasks=1): 112 | if ntasks == len(self._tasks): 113 | result = list(self._tasks) 114 | self._tasks.clear() 115 | else: 116 | result = [self._tasks.pop() for _ in range(ntasks)] 117 | return result 118 | 119 | async def wake(self, n=None): 120 | ''' 121 | Wake all or a specified number of tasks. 122 | ''' 123 | n = len(self._tasks) if n is None else n 124 | await _scheduler_wake(self, n) 125 | 126 | -------------------------------------------------------------------------------- /concorrencia/curio-example/curio/socket.py: -------------------------------------------------------------------------------- 1 | # curio/socket.py 2 | # 3 | # Standin for the standard socket library. The entire contents of stdlib socket are 4 | # made available here. However, the socket class is replaced by an async compatible version. 5 | # Certain blocking operations are also replaced by versions safe to use in async. 6 | # 7 | 8 | import socket as _socket 9 | 10 | __all__ = _socket.__all__ 11 | 12 | from socket import * 13 | from functools import wraps, partial 14 | 15 | from . import workers 16 | from . import io 17 | 18 | 19 | @wraps(_socket.socket) 20 | def socket(*args, **kwargs): 21 | return io.Socket(_socket.socket(*args, **kwargs)) 22 | 23 | 24 | @wraps(_socket.socketpair) 25 | def socketpair(*args, **kwargs): 26 | s1, s2 = _socket.socketpair(*args, **kwargs) 27 | return io.Socket(s1), io.Socket(s2) 28 | 29 | 30 | @wraps(_socket.fromfd) 31 | def fromfd(*args, **kwargs): 32 | return io.Socket(_socket.fromfd(*args, **kwargs)) 33 | 34 | # Replacements for blocking functions related to domain names and DNS 35 | 36 | #@wraps(_socket.create_connection) 37 | #async def create_connection(*args, **kwargs): 38 | # sock = await workers.run_in_thread(partial(_socket.create_connection, *args, **kwargs)) 39 | # return io.Socket(sock) 40 | 41 | async def create_connection(address, timeout=None, source_address=None): 42 | ''' 43 | Pure async implementation of the socket.create_connection function in standard library 44 | ''' 45 | host, port = address 46 | err = None 47 | for res in await getaddrinfo(host, port, 0, SOCK_STREAM): 48 | af, socktype, proto, canonname, sa = res 49 | sock = None 50 | try: 51 | sock = socket(af, socktype, proto) 52 | if source_address: 53 | sock.bind(source_address) 54 | await sock.connect(sa) 55 | # Break explicitly a reference cycle 56 | err = None 57 | return sock 58 | 59 | except error as _: 60 | err = _ 61 | if sock is not None: 62 | await sock.close() 63 | 64 | if err is not None: 65 | raise err 66 | else: 67 | raise OSError("getaddrinfo returns an empty list") 68 | 69 | @wraps(_socket.getaddrinfo) 70 | async def getaddrinfo(*args, **kwargs): 71 | return await workers.run_in_thread(partial(_socket.getaddrinfo, *args, **kwargs)) 72 | 73 | 74 | @wraps(_socket.getfqdn) 75 | async def getfqdn(*args, **kwargs): 76 | return await workers.run_in_thread(partial(_socket.getfqdn, *args, **kwargs)) 77 | 78 | 79 | @wraps(_socket.gethostbyname) 80 | async def gethostbyname(*args, **kwargs): 81 | return await workers.run_in_thread(partial(_socket.gethostbyname, *args, **kwargs)) 82 | 83 | 84 | @wraps(_socket.gethostbyname_ex) 85 | async def gethostbyname_ex(*args, **kwargs): 86 | return await workers.run_in_thread(partial(_socket.gethostbyname_ex, *args, **kwargs)) 87 | 88 | 89 | @wraps(_socket.gethostname) 90 | async def gethostname(*args, **kwargs): 91 | return await workers.run_in_thread(partial(_socket.gethostname, *args, **kwargs)) 92 | 93 | 94 | @wraps(_socket.gethostbyaddr) 95 | async def gethostbyaddr(*args, **kwargs): 96 | return await workers.run_in_thread(partial(_socket.gethostbyaddr, *args, **kwargs)) 97 | 98 | 99 | @wraps(_socket.getnameinfo) 100 | async def getnameinfo(*args, **kwargs): 101 | return await workers.run_in_thread(partial(_socket.getnameinfo, *args, **kwargs)) 102 | -------------------------------------------------------------------------------- /concorrencia/curio-example/curio/ssl.py: -------------------------------------------------------------------------------- 1 | # curio/ssl.py 2 | # 3 | # Wrapper around built-in SSL module 4 | 5 | __all__ = [] 6 | 7 | # -- Standard Library 8 | 9 | from functools import wraps, partial 10 | 11 | try: 12 | import ssl as _ssl 13 | from ssl import * 14 | except ImportError: 15 | _ssl = None 16 | 17 | # We need these exceptions defined, even if ssl is not available. 18 | class SSLWantReadError(Exception): 19 | pass 20 | 21 | class SSLWantWriteError(Exception): 22 | pass 23 | 24 | # -- Curio 25 | 26 | from .workers import run_in_thread 27 | from .io import Socket 28 | 29 | if _ssl: 30 | @wraps(_ssl.SSLContext.wrap_socket) 31 | async def wrap_socket(sock, *args, do_handshake_on_connect=True, **kwargs): 32 | if isinstance(sock, Socket): 33 | sock = sock._socket 34 | 35 | ssl_sock = _ssl.SSLContext.wrap_socket(sock, *args, do_handshake_on_connect=False, **kwargs) 36 | cssl_sock = Socket(ssl_sock) 37 | cssl_sock.do_handshake_on_connect = do_handshake_on_connect 38 | if do_handshake_on_connect and ssl_sock._connected: 39 | await cssl_sock.do_handshake() 40 | return cssl_sock 41 | 42 | @wraps(_ssl.get_server_certificate) 43 | async def get_server_certificate(*args, **kwargs): 44 | return await run_in_thread(partial(_ssl.get_server_certificate, *args, **kwargs)) 45 | 46 | # Small wrapper class to make sure the wrap_socket() method returns the right type 47 | class CurioSSLContext(object): 48 | 49 | def __init__(self, context): 50 | self._context = context 51 | 52 | def __getattr__(self, name): 53 | return getattr(self._context, name) 54 | 55 | async def wrap_socket(self, sock, *args, do_handshake_on_connect=True, **kwargs): 56 | sock = self._context.wrap_socket( 57 | sock._socket, *args, do_handshake_on_connect=False, **kwargs) 58 | csock = Socket(sock) 59 | csock.do_handshake_on_connect = do_handshake_on_connect 60 | if do_handshake_on_connect and sock._connected: 61 | await csock.do_handshake() 62 | return csock 63 | 64 | def __setattr__(self, name, value): 65 | if name == '_context': 66 | super().__setattr__(name, value) 67 | else: 68 | setattr(self._context, name, value) 69 | 70 | # Name alias 71 | def SSLContext(protocol): 72 | return CurioSSLContext(_ssl.SSLContext(protocol)) 73 | 74 | @wraps(_ssl.create_default_context) 75 | def create_default_context(*args, **kwargs): 76 | context = _ssl.create_default_context(*args, **kwargs) 77 | return CurioSSLContext(context) 78 | -------------------------------------------------------------------------------- /concorrencia/curio-example/curio/timequeue.py: -------------------------------------------------------------------------------- 1 | # timequeue.py 2 | # 3 | # A Discussion About Time. 4 | # 5 | # Internally, Curio must manage time for two different reasons, 6 | # sleeping and for timeouts. Aside from toy examples, most real-world 7 | # code isn't going to sit around making a lot of sleep() calls. 8 | # Instead the more common use is timeouts. Timeouts are kind of 9 | # interesting though--when a timeout is set, there is typically an 10 | # expectation that it will probably NOT occur. The expiration of a 11 | # timeout is an exceptional event. Most of the time, a timeout will be 12 | # cancelled before it is allowed to expire. 13 | # 14 | # This presents an interesting implementation challenge for managing 15 | # time. It is most common to see time managed by sorting the 16 | # expiration times in some way. For example, placing them in a sorted 17 | # list, or ordering them on a heap in a priority queue. Although 18 | # operations on these kinds of data structures can be managed in O(log N) 19 | # steps, they might not be necessary at all if you make some slightly 20 | # different assumptions about time management. 21 | # 22 | # The queue implementation here is based on the idea that expiration 23 | # times in the distant future don't need to be precisely sorted. 24 | # Instead, you can merely drop expiration times in a dict with the 25 | # hope that they'll be cancelled later. Manipulating a dict in this 26 | # case is O(1)--meaning that is extremely cheap to setup and teardown 27 | # a timeout that never occurs. For timeouts in the near future, they 28 | # can still be sorted using a priority queue in the usual way. 29 | 30 | import heapq 31 | 32 | class TimeQueue: 33 | cutoff = 1.0 # Threshhold for near/far events (seconds) 34 | def __init__(self): 35 | self.near = [ ] 36 | self.far = { } 37 | self.near_deadline = 0 38 | self.far_min_deadline = float('inf') 39 | 40 | def _far_to_near(self): 41 | ''' 42 | Move items from the far queue to the near queue (if any). 43 | ''' 44 | removed = [] 45 | min_deadline = float('inf') 46 | for item, expires in self.far.items(): 47 | if expires < self.near_deadline: 48 | self.push(item, expires) 49 | removed.append(item) 50 | elif expires < min_deadline: 51 | min_deadline = expires 52 | for item in removed: 53 | del self.far[item] 54 | self.far_min_deadline = min_deadline 55 | 56 | def next_deadline(self, current_clock): 57 | ''' 58 | Returns the number of seconds to delay until the next deadline 59 | expires. current_clock is the current value of the clock. 60 | Returns None if there are no pending deadlines. 61 | ''' 62 | self.near_deadline = current_clock + self.cutoff 63 | if self.near_deadline > self.far_min_deadline: 64 | self._far_to_near() 65 | 66 | if self.near: 67 | delta = self.near[0][0] - current_clock 68 | return delta if delta > 0 else 0 69 | 70 | # There are no near deadlines. Use the closest far deadline 71 | if self.far: 72 | delta = self.far_min_deadline - current_clock 73 | return delta if delta > 0 else 0 74 | 75 | # There are no sleeping tasks of any kind. 76 | return None 77 | 78 | def push(self, item, expires): 79 | ''' 80 | Push a new item onto the time queue. 81 | ''' 82 | # If the expiration time is closer than the current near deadline, 83 | # it gets pushed onto a heap in order to preserve order 84 | if expires <= self.near_deadline: 85 | heapq.heappush(self.near, (expires, item)) 86 | else: 87 | # Otherwise the item gets put into a dict for far-in-future handling 88 | if item not in self.far or self.far[item] > expires: 89 | self.far[item] = expires 90 | if expires < self.far_min_deadline: 91 | self.far_min_deadline = expires 92 | 93 | def expired(self, deadline): 94 | ''' 95 | An iterator that returns all items that have expired up to a given deadline 96 | ''' 97 | near = self.near 98 | if deadline >= self.far_min_deadline: 99 | self.near_deadline = deadline + self.cutoff 100 | self._far_to_near() 101 | 102 | while near and near[0][0] <= deadline: 103 | yield heapq.heappop(near) 104 | 105 | def cancel(self, item, expires): 106 | ''' 107 | Cancel a time event. The combination of (item, expires) should 108 | match a prior push() operation (but if not, it's ignored). 109 | ''' 110 | self.far.pop(item, None) 111 | -------------------------------------------------------------------------------- /concorrencia/curio-example/curio/traps.py: -------------------------------------------------------------------------------- 1 | # traps.py 2 | # 3 | # Curio programs execute under the supervision of a 4 | # kernel. Communication with the kernel takes place via a "trap" 5 | # involving the yield statement. Traps represent internel kernel 6 | # procedures. Direct use of the functions defined here is allowed 7 | # when making new kinds of Curio primitives, but if you're trying to 8 | # solve a higher level problem, there is probably a higher-level 9 | # interface that is easier to use (e.g., Socket, File, Queue, etc.). 10 | # ---------------------------------------------------------------------- 11 | 12 | __all__ = [ 13 | '_read_wait', '_write_wait', '_future_wait', '_sleep', '_spawn', 14 | '_cancel_task', '_scheduler_wait', '_scheduler_wake', 15 | '_get_kernel', '_get_current', '_set_timeout', '_unset_timeout', 16 | '_clock', '_io_waiting', '_io_release', 17 | ] 18 | 19 | # -- Standard library 20 | 21 | from types import coroutine 22 | from selectors import EVENT_READ, EVENT_WRITE 23 | 24 | # -- Curio 25 | 26 | from . import errors 27 | 28 | # This is the only entry point to the Curio kernel and the 29 | # only place where the @types.coroutine decorator is used. 30 | @coroutine 31 | def _kernel_trap(*request): 32 | result = yield request 33 | if isinstance(result, BaseException): 34 | raise result 35 | else: 36 | return result 37 | 38 | # Higher-level trap functions that make use of async/await 39 | async def _read_wait(fileobj): 40 | ''' 41 | Wait until reading can be performed. If another task is waiting 42 | on the same file, a ResourceBusy exception is raised. 43 | ''' 44 | return await _kernel_trap('trap_io', fileobj, EVENT_READ, 'READ_WAIT') 45 | 46 | async def _write_wait(fileobj): 47 | ''' 48 | Wait until writing can be performed. If another task is waiting 49 | to write on the same file, a ResourceBusy exception is raised. 50 | ''' 51 | return await _kernel_trap('trap_io', fileobj, EVENT_WRITE, 'WRITE_WAIT') 52 | 53 | async def _io_release(fileobj): 54 | ''' 55 | Release kernel resources associated with a file 56 | ''' 57 | return await _kernel_trap('trap_io_release', fileobj) 58 | 59 | async def _io_waiting(fileobj): 60 | ''' 61 | Return a tuple (rtask, wtask) of tasks currently blocked waiting 62 | for I/O on fileobj. 63 | ''' 64 | return await _kernel_trap('trap_io_waiting', fileobj) 65 | 66 | async def _future_wait(future, event=None): 67 | ''' 68 | Wait for the result of a Future to be ready. 69 | ''' 70 | return await _kernel_trap('trap_future_wait', future, event) 71 | 72 | async def _sleep(clock): 73 | ''' 74 | Sleep until the monotonic clock reaches the specified clock value. 75 | If clock is 0, forces the current task to yield to the next task (if any). 76 | ''' 77 | return await _kernel_trap('trap_sleep', clock) 78 | 79 | async def _spawn(coro): 80 | ''' 81 | Create a new task. Returns the resulting Task object. 82 | ''' 83 | return await _kernel_trap('trap_spawn', coro) 84 | 85 | async def _cancel_task(task, exc=errors.TaskCancelled, val=None): 86 | ''' 87 | Cancel a task. Causes a CancelledError exception to raise in the task. 88 | Set the exc and val arguments to change the exception. 89 | ''' 90 | return await _kernel_trap('trap_cancel_task', task, exc, val) 91 | 92 | async def _scheduler_wait(sched, state): 93 | ''' 94 | Put the task to sleep on a scheduler primitive. 95 | ''' 96 | return await _kernel_trap('trap_sched_wait', sched, state) 97 | 98 | async def _scheduler_wake(sched, n=1): 99 | ''' 100 | Reschedule one or more tasks waiting on a scheduler primitive. 101 | ''' 102 | return await _kernel_trap('trap_sched_wake', sched, n) 103 | 104 | async def _get_kernel(): 105 | ''' 106 | Get the kernel executing the task. 107 | ''' 108 | return await _kernel_trap('trap_get_kernel') 109 | 110 | async def _get_current(): 111 | ''' 112 | Get the currently executing task 113 | ''' 114 | return await _kernel_trap('trap_get_current') 115 | 116 | async def _set_timeout(clock): 117 | ''' 118 | Set a timeout for the current task that occurs at the specified clock value. 119 | Setting a clock of None clears any previous timeout. 120 | ''' 121 | return await _kernel_trap('trap_set_timeout', clock) 122 | 123 | async def _unset_timeout(previous): 124 | ''' 125 | Restore the previous timeout for the current task. 126 | ''' 127 | return await _kernel_trap('trap_unset_timeout', previous) 128 | 129 | async def _clock(): 130 | ''' 131 | Return the value of the kernel clock 132 | ''' 133 | return await _kernel_trap('trap_clock') 134 | -------------------------------------------------------------------------------- /concorrencia/do-zero-pronto/blogdom_asyncio.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import asyncio 3 | import socket 4 | from keyword import kwlist, softkwlist 5 | 6 | MAX_KEYWORD_LEN = 5 7 | NAMES = [kw for kw in kwlist + softkwlist if len(kw) <= MAX_KEYWORD_LEN] 8 | 9 | 10 | async def probe(domain: str) -> tuple[str, bool]: 11 | try: 12 | await asyncio.get_running_loop().getaddrinfo(domain, None) 13 | except socket.gaierror: 14 | return (domain, False) 15 | return (domain, True) 16 | 17 | 18 | async def main(): 19 | domains = (f'{name}.dev'.lower() for name in NAMES) 20 | coros = [probe(domain) for domain in domains] # (8) 21 | for coro in asyncio.as_completed(coros): # (9) 22 | domain, found = await coro # (10) 23 | mark = '+' if found else ' ' 24 | print(f'{mark} {domain}') 25 | 26 | 27 | if __name__ == '__main__': 28 | asyncio.run(main()) 29 | -------------------------------------------------------------------------------- /concorrencia/do-zero-pronto/blogdom_curio.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from curio import run, TaskGroup 3 | import curio.socket as socket 4 | from keyword import kwlist, softkwlist 5 | 6 | MAX_KEYWORD_LEN = 5 7 | NAMES = [kw for kw in kwlist + softkwlist if len(kw) <= MAX_KEYWORD_LEN] 8 | 9 | 10 | async def probe(domain: str) -> tuple[str, bool]: # (1) 11 | try: 12 | await socket.getaddrinfo(domain, None) # (2) 13 | except socket.gaierror: 14 | return (domain, False) 15 | return (domain, True) 16 | 17 | 18 | async def main(): 19 | domains = (f'{name}.dev'.lower() for name in NAMES) 20 | async with TaskGroup() as group: # (3) 21 | for domain in domains: 22 | await group.spawn(probe, domain) # (4) 23 | async for task in group: # (5) 24 | domain, found = task.result 25 | mark = '+' if found else ' ' 26 | print(f'{mark} {domain}') 27 | 28 | 29 | if __name__ == '__main__': 30 | run(main()) # (6) 31 | -------------------------------------------------------------------------------- /concorrencia/do-zero-pronto/blogdom_trio.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import trio 3 | import trio.socket as socket 4 | from trio.abc import SendChannel, ReceiveChannel 5 | from keyword import kwlist, softkwlist 6 | 7 | MAX_KEYWORD_LEN = 5 8 | NAMES = [kw for kw in kwlist + softkwlist if len(kw) <= MAX_KEYWORD_LEN] 9 | 10 | 11 | async def probe(sender: SendChannel, domain: str): 12 | async with sender: 13 | try: 14 | await socket.getaddrinfo(domain, None) 15 | result = (domain, True) 16 | except socket.gaierror: 17 | result = (domain, False) 18 | await sender.send(result) 19 | 20 | 21 | async def report(receiver: ReceiveChannel): 22 | async with receiver: 23 | async for domain, found in receiver: 24 | mark = '+' if found else '-' 25 | print(f'{mark} {domain}') 26 | 27 | 28 | async def main(): 29 | domains = [f'{name}.dev'.lower() for name in NAMES] 30 | sender, receiver = trio.open_memory_channel(0) 31 | 32 | async with trio.open_nursery() as nursery: 33 | async with sender: 34 | for domain in domains: 35 | nursery.start_soon(probe, sender.clone(), domain) 36 | 37 | nursery.start_soon(report, receiver) 38 | 39 | 40 | if __name__ == '__main__': 41 | trio.run(main) 42 | -------------------------------------------------------------------------------- /concorrencia/do-zero-pronto/curio: -------------------------------------------------------------------------------- 1 | /Users/luciano/src/curio/curio -------------------------------------------------------------------------------- /concorrencia/do-zero-pronto/gira_async.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import asyncio 3 | import time 4 | 5 | 6 | async def girar(msg: str) -> None: 7 | for char in itertools.cycle(r'\|/-'): 8 | status = f'{char} {msg}' 9 | print(status, end='\r', flush=True) 10 | # if calculado.wait(0.05): 11 | # break 12 | try: 13 | await asyncio.sleep(0.05) 14 | except asyncio.CancelledError: 15 | break 16 | blanks = ' ' * len(status) 17 | print(f'\r{blanks}\r', end='', flush=True) 18 | 19 | 20 | async def buscar() -> int: 21 | await asyncio.sleep(3) 22 | return 42 23 | 24 | 25 | async def main(): 26 | # girador = Thread(target=girar, args=['pensando...', calculado]) 27 | girador = asyncio.create_task(girar('pensando...')) 28 | # girador.start() 29 | res = await buscar() 30 | girador.cancel() 31 | return res 32 | 33 | 34 | if __name__ == '__main__': 35 | res = asyncio.run(main()) 36 | print('Resposta:', res) 37 | -------------------------------------------------------------------------------- /concorrencia/do-zero-pronto/gira_proc.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import time 3 | 4 | # from threading import Thread, Event 5 | from multiprocessing import Process, Event 6 | 7 | 8 | def girar(msg: str, pronto: Event) -> None: 9 | for char in itertools.cycle(r'\|/-'): 10 | status = f'\r{char} {msg}' 11 | print(status, end='', flush=True) 12 | if pronto.wait(0.05): 13 | break 14 | blanks = ' ' * len(status) 15 | print(f'\r{blanks}\r', end='') 16 | 17 | 18 | def buscar() -> int: 19 | time.sleep(3) 20 | return 42 21 | 22 | 23 | def main(): 24 | pronto = Event() 25 | # fio = Thread(target=girar, args=['pensando...', pronto]) 26 | fio = Process(target=girar, args=['pensando...', pronto]) 27 | fio.start() 28 | res = buscar() 29 | pronto.set() 30 | fio.join() 31 | print(res) 32 | 33 | 34 | if __name__ == '__main__': 35 | main() 36 | -------------------------------------------------------------------------------- /concorrencia/do-zero-pronto/gira_sec.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import time 3 | 4 | 5 | def girar(msg: str) -> None: 6 | for char in itertools.cycle(r'\|/-'): 7 | status = f'\r{char} {msg}' 8 | print(status, end='', flush=True) 9 | time.sleep(0.1) 10 | 11 | 12 | def main() -> None: 13 | try: 14 | girar('pensando para sempre...') 15 | except KeyboardInterrupt: 16 | pass 17 | 18 | 19 | if __name__ == '__main__': 20 | main() 21 | -------------------------------------------------------------------------------- /concorrencia/do-zero-pronto/gira_sec_braille.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import time 3 | from unicodedata import lookup 4 | 5 | 6 | # ref: https://en.wikipedia.org/wiki/Braille_Patterns 7 | DOTS = reversed((1, 2, 3, 7, 8, 6, 5, 4)) 8 | PREFIX = 'BRAILLE PATTERN DOTS-' 9 | 10 | 11 | def girar(msg: str) -> None: 12 | chars = (lookup(PREFIX + str(dot)) for dot in DOTS) 13 | for char in itertools.cycle(chars): 14 | status = f'\r{char} {msg}' 15 | print(status, end='', flush=True) 16 | time.sleep(0.1) 17 | 18 | 19 | def main() -> None: 20 | try: 21 | girar('pensando para sempre...') 22 | except KeyboardInterrupt: 23 | pass 24 | 25 | 26 | if __name__ == '__main__': 27 | main() 28 | -------------------------------------------------------------------------------- /concorrencia/do-zero-pronto/gira_thread.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import time 3 | from threading import Thread, Event 4 | 5 | 6 | def girar(msg: str, calculado: Event) -> None: 7 | for char in itertools.cycle(r'\|/-'): 8 | status = f'\r{char} {msg}' 9 | print(status, end='', flush=True) 10 | if calculado.wait(0.05): 11 | break 12 | blanks = ' ' * len(status) 13 | print(f'\r{blanks}\r', end='') 14 | 15 | 16 | def calcular() -> int: 17 | time.sleep(3) 18 | return 42 19 | 20 | 21 | def main(): 22 | calculado = Event() 23 | fio = Thread(target=girar, args=['pensando...', calculado]) 24 | fio.start() 25 | res = calcular() 26 | calculado.set() 27 | fio.join() 28 | print(res) 29 | 30 | 31 | if __name__ == '__main__': 32 | main() 33 | -------------------------------------------------------------------------------- /concorrencia/do-zero/gira_proc.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import time 3 | #from threading import Thread, Event 4 | from multiprocessing import Process, Event 5 | 6 | def girar(msg: str, pronto: Event) -> None: 7 | for char in itertools.cycle(r'\|/-'): 8 | status = f'\r{char} {msg}' 9 | print(status, end='', flush=True) 10 | if pronto.wait(0.05): 11 | break 12 | blanks = ' ' * len(status) 13 | print(f'\r{blanks}\r', end='') 14 | 15 | 16 | def buscar() -> int: 17 | time.sleep(3) 18 | return 42 19 | 20 | 21 | def main(): 22 | pronto = Event() 23 | #fio = Thread(target=girar, args=['pensando...', pronto]) 24 | fio = Process(target=girar, args=['pensando...', pronto]) 25 | fio.start() 26 | res = buscar() 27 | pronto.set() 28 | fio.join() 29 | print(res) 30 | 31 | if __name__ == '__main__': 32 | main() 33 | -------------------------------------------------------------------------------- /concorrencia/do-zero/notebook_components.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramalho/python-eng/59a007d54595f11441ec7de07909576e1c158772/concorrencia/do-zero/notebook_components.png -------------------------------------------------------------------------------- /concorrencia/do-zero/ola-mundo-concorrente.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "bf4f7db9-ab8f-40c6-8890-8c85fca5503b", 6 | "metadata": {}, 7 | "source": [ 8 | "# Olá Mundo Concorrente\n", 9 | "\n", 10 | "Este notebook está em https://github.com/ramalho/python-eng/blob/main/concorrencia/do-zero/ola-mundo-concorrente.ipynb\n", 11 | "\n", 12 | "## Como animar texto na saída padrão\n", 13 | "\n", 14 | "Imagine um **teletipo** ([teletype](http://www.columbia.edu/cu/computinghistory/teletype/index.html)): uma máquina de escrever automatizada que pode trocar mensagens via modem ou comunicação serial com outro teletipo ou com um computador.\n", 15 | "\n", 16 | "\"Foto\n", 17 | "\n", 18 | "Por isso a tabela ASCII inclui códigos de controle como estes:\n", 19 | "\n", 20 | "|dec|hex |nome |sigla|Python|\n", 21 | "|--:|----|---------------|:---:|------|\n", 22 | "| 7|0x07|bell (alarm) |BEL |`'\\a'`|\n", 23 | "| 8|0x08|backspace |BS |`'\\b'`|\n", 24 | "| 9|0x09|horizontal tab |HT |`'\\t'`|\n", 25 | "| 10|0x0a|line feed |LF |`'\\n'`|\n", 26 | "| 11|0x0b|vertical tab |VT |`'\\v'`|\n", 27 | "| 12|0x0c|form feed |FF |`'\\f'`|\n", 28 | "| 13|0x0d|carriage return|CR |`'\\r'`|\n", 29 | "\n", 30 | "**Nota:** Nem todos esses códigos funcionam no Jupyter Notebook ou em alguns terminais." 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": null, 36 | "id": "330f9db5-9499-43dd-8e65-99ad00f00cf6", 37 | "metadata": {}, 38 | "outputs": [], 39 | "source": [ 40 | "import time\n", 41 | "\n", 42 | "def relogio():\n", 43 | " while True:\n", 44 | " print(time.strftime('%H:%M:%S'), end='\\r')\n", 45 | " time.sleep(1)\n", 46 | "\n", 47 | "# relogio()" 48 | ] 49 | }, 50 | { 51 | "cell_type": "markdown", 52 | "id": "ac02d2c1-926a-46c7-9fcf-7d247d73c2a9", 53 | "metadata": {}, 54 | "source": [ 55 | "## Exemplo com threads\n", 56 | "\n", 57 | "Duas threads: a thread principal e aquela nós criamos explicitamente." 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": 9, 63 | "id": "c052a707-dc73-4370-8fe4-42a5284b87d3", 64 | "metadata": {}, 65 | "outputs": [ 66 | { 67 | "name": "stdout", 68 | "output_type": "stream", 69 | "text": [ 70 | "Resposta: 42 \n" 71 | ] 72 | } 73 | ], 74 | "source": [ 75 | "import itertools\n", 76 | "import time\n", 77 | "from threading import Thread, Event\n", 78 | "\n", 79 | "def girar(msg, pronto):\n", 80 | " for car in itertools.cycle('|/-\\\\'):\n", 81 | " status = f'{car} {msg}'\n", 82 | " print(status, end='\\r')\n", 83 | " if pronto.wait(.5):\n", 84 | " break\n", 85 | " brancos = ' ' * len(status)\n", 86 | " print(f'\\r{brancos}', end='\\r')\n", 87 | "\n", 88 | "def buscar():\n", 89 | " time.sleep(3)\n", 90 | " return 42\n", 91 | "\n", 92 | "def main():\n", 93 | " pronto = Event()\n", 94 | " giradora = Thread(target=girar, args=['buscando a resposta para a pergunta mais importante...', pronto])\n", 95 | " giradora.start()\n", 96 | " res = buscar()\n", 97 | " pronto.set()\n", 98 | " giradora.join()\n", 99 | " print('Resposta:', res)\n", 100 | "\n", 101 | "main()" 102 | ] 103 | }, 104 | { 105 | "cell_type": "markdown", 106 | "id": "897b3bf2-3c99-4115-830a-9323b07db299", 107 | "metadata": {}, 108 | "source": [ 109 | "## Exemplos com processos ou corrotinas\n", 110 | "\n", 111 | "As versões com processos (`multiprocessing`) ou corrotinas (`asyncio`) não funcionam aqui por diferentes motivos ligados à\n", 112 | "[arquitetura do no Jupyter Notebook](https://docs.jupyter.org/en/latest/projects/architecture/content-architecture.html#the-jupyter-notebook-interface).\n", 113 | "\n", 114 | "\n", 115 | "\n", 116 | "**Dica:** Rode no terminal os programas `gira_proc.py` e `gira_async.py`." 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": null, 122 | "id": "7a0243ec-1f56-44fe-a587-b285da75bb15", 123 | "metadata": {}, 124 | "outputs": [], 125 | "source": [] 126 | } 127 | ], 128 | "metadata": { 129 | "kernelspec": { 130 | "display_name": "Python 3 (ipykernel)", 131 | "language": "python", 132 | "name": "python3" 133 | }, 134 | "language_info": { 135 | "codemirror_mode": { 136 | "name": "ipython", 137 | "version": 3 138 | }, 139 | "file_extension": ".py", 140 | "mimetype": "text/x-python", 141 | "name": "python", 142 | "nbconvert_exporter": "python", 143 | "pygments_lexer": "ipython3", 144 | "version": "3.12.2" 145 | } 146 | }, 147 | "nbformat": 4, 148 | "nbformat_minor": 5 149 | } 150 | -------------------------------------------------------------------------------- /concorrencia/do-zero/pcworld-sel.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramalho/python-eng/59a007d54595f11441ec7de07909576e1c158772/concorrencia/do-zero/pcworld-sel.jpg -------------------------------------------------------------------------------- /concorrencia/do-zero/tty33.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramalho/python-eng/59a007d54595f11441ec7de07909576e1c158772/concorrencia/do-zero/tty33.jpg -------------------------------------------------------------------------------- /concorrencia/executors/demo_executor_map.py: -------------------------------------------------------------------------------- 1 | """ 2 | Experiment with ``ThreadPoolExecutor.map`` 3 | """ 4 | # tag::EXECUTOR_MAP[] 5 | from time import sleep, strftime 6 | from concurrent import futures 7 | 8 | def display(*args): # <1> 9 | print(strftime('[%H:%M:%S]'), end=' ') 10 | print(*args) 11 | 12 | def loiter(n): # <2> 13 | msg = '{}loiter({}): doing nothing for {}s...' 14 | display(msg.format('\t'*n, n, n)) 15 | sleep(n) 16 | msg = '{}loiter({}): done.' 17 | display(msg.format('\t'*n, n)) 18 | return n * 10 # <3> 19 | 20 | def main(): 21 | display('Script starting.') 22 | executor = futures.ThreadPoolExecutor(max_workers=3) # <4> 23 | results = executor.map(loiter, range(5)) # <5> 24 | display('results:', results) # <6> 25 | display('Waiting for individual results:') 26 | for i, result in enumerate(results): # <7> 27 | display(f'result {i}: {result}') 28 | 29 | if __name__ == '__main__': 30 | main() 31 | # end::EXECUTOR_MAP[] 32 | -------------------------------------------------------------------------------- /concorrencia/executors/getflags/.gitignore: -------------------------------------------------------------------------------- 1 | flags/ 2 | downloaded/ -------------------------------------------------------------------------------- /concorrencia/executors/getflags/README.adoc: -------------------------------------------------------------------------------- 1 | = Experimenting with the `flags2*` examples 2 | 3 | The `flags2*` examples enhance the `flags*` examples with error handling and reporting. 4 | Therefore, we need a server that generates errors and delays to experiment with them. 5 | 6 | The main reason for these instructions is to document how to configure one such server 7 | in your machine, and how to tell the `flags2*` clients to access it. 8 | The other reason is to alert of an installation step that MacOS users sometimes overlook. 9 | 10 | Contents: 11 | 12 | * <> 13 | * <> 14 | * <> 15 | 16 | [[server_setup]] 17 | == Setting up test servers 18 | 19 | If you don't already have a local HTTP server for testing, 20 | here are the steps to experiment with the `flags2*` examples 21 | using just the Python ≥ 3.9 distribution: 22 | 23 | . Clone or download the https://github.com/fluentpython/example-code-2e[_Fluent Python 2e_ code repository] (this repo!). 24 | . Open your shell and go to the _20-futures/getflags/_ directory of your local copy of the repository (this directory!) 25 | . Unzip the _flags.zip_ file, creating a _flags_ directory at _20-futures/getflags/flags/_. 26 | . Open a second shell, go to the _20-futures/getflags/_ directory and run `python3 -m http.server`. This will start a `ThreadingHTTPServer` listening to port 8000, serving the local files. If you open the URL http://localhost:8000/flags/[http://localhost:8000/flags/] with your browser, you'll see a long list of directories named with two-letter country codes from `ad/` to `zw/`. 27 | . Now you can go back to the first shell and run the _flags2*.py_ examples with the default `--server LOCAL` option. 28 | . To test with the `--server DELAY` option, go to _20-futures/getflags/_ and run `python3 slow_server.py`. This binds to port 8001 by default. It will add a random delay of .5s to 5s before each response. 29 | . To test with the `--server ERROR` option, go to _20-futures/getflags/_ and run `python3 slow_server.py 8002 --error-rate .25`. 30 | Each request will have a 25% probability of getting a 31 | https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/418[418 I'm a teapot] response, 32 | and all responses will be delayed .5s. 33 | 34 | I wrote _slow_server.py_ reusing code from Python's 35 | https://github.com/python/cpython/blob/917eca700aa341f8544ace43b75d41b477e98b72/Lib/http/server.py[`http.server`] standard library module, 36 | which "is not recommended for production"—according to the 37 | https://docs.python.org/3/library/http.server.html[documentation]. 38 | 39 | [NOTE] 40 | ==== 41 | This is a simple testing environment that does not require any external libraries or 42 | tools—apart from the libraries used in the `flags2*` scripts themselves, as discussed in the book. 43 | 44 | For a more robust testing environment, I recommend configuring 45 | https://www.nginx.com/[NGINX] and 46 | https://github.com/shopify/toxiproxy[Toxiproxy] with equivalent parameters. 47 | ==== 48 | 49 | [[client_setup]] 50 | == Running a `flags2*` script 51 | 52 | The `flags2*` examples provide a command-line interface. 53 | All three scripts accept the same options, 54 | and you can see them by running any of the scripts with the `-h` option: 55 | 56 | [[flags2_help_demo]] 57 | .Help screen for the scripts in the flags2 series 58 | ==== 59 | [source, text] 60 | ---- 61 | $ python3 flags2_threadpool.py -h 62 | usage: flags2_threadpool.py [-h] [-a] [-e] [-l N] [-m CONCURRENT] [-s LABEL] 63 | [-v] 64 | [CC [CC ...]] 65 | 66 | Download flags for country codes. Default: top 20 countries by population. 67 | 68 | positional arguments: 69 | CC country code or 1st letter (eg. B for BA...BZ) 70 | 71 | optional arguments: 72 | -h, --help show this help message and exit 73 | -a, --all get all available flags (AD to ZW) 74 | -e, --every get flags for every possible code (AA...ZZ) 75 | -l N, --limit N limit to N first codes 76 | -m CONCURRENT, --max_req CONCURRENT 77 | maximum concurrent requests (default=30) 78 | -s LABEL, --server LABEL 79 | Server to hit; one of DELAY, ERROR, LOCAL, REMOTE 80 | (default=LOCAL) 81 | -v, --verbose output detailed progress info 82 | 83 | ---- 84 | ==== 85 | 86 | All arguments are optional. The most important arguments are discussed next. 87 | 88 | One option you can't ignore is `-s/--server`: it lets you choose which HTTP server and base URL will be used in the test. 89 | You can pass one of four labels to determine where the script will look for the flags (the labels are case-insensitive): 90 | 91 | `LOCAL`:: Use `http://localhost:8000/flags`; this is the default. 92 | You should configure a local HTTP server to answer at port 8000. See <> for instructions. 93 | Feel free to hit this as hard as you can. It's your machine! 94 | 95 | `REMOTE`:: Use `http://fluentpython.com/data/flags`; that is a public website owned by me, hosted on a shared server. 96 | Please do not hit it with too many concurrent requests. 97 | The `fluentpython.com` domain is handled by the http://www.cloudflare.com/[Cloudflare] CDN (Content Delivery Network) 98 | so you may notice that the first downloads are slower, but they get faster when the CDN cache warms 99 | up.footnote:[Before configuring Cloudflare, I got HTTP 503 errors--Service Temporarily Unavailable--when 100 | testing the scripts with a few dozen concurrent requests on my inexpensive shared host account. Now those errors are gone.] 101 | 102 | `DELAY`:: Use `http://localhost:8001/flags`; a server delaying HTTP responses should be listening to port 8001. 103 | I wrote _slow_server.py_ to make it easier to experiment. See <> for instructions. 104 | 105 | `ERROR`:: Use `http://localhost:8002/flags`; a server introducing HTTP errors and delaying responses should be installed at port 8002. 106 | Running _slow_server.py_ is an easy way to do it. See <>. 107 | 108 | [[macos_certificates]] 109 | == Install SSL Certificates (for MacOS) 110 | 111 | On Macos, depending on how you installed Python you may need to manually run a command 112 | after Python's installer finishes, to install the SSL certificates Python uses to make HTTPS connections. 113 | 114 | Using the Finder, open the `Python 3.X` folder inside `/Applications` folder 115 | and double-click "Install Certificates" or "Install Certificates.command". 116 | 117 | Using the terminal, you can type for example: 118 | 119 | [source, text] 120 | ---- 121 | $ open /Applications/Python 3.10/"Install Certificates.command" 122 | ---- 123 | -------------------------------------------------------------------------------- /concorrencia/executors/getflags/country_codes.txt: -------------------------------------------------------------------------------- 1 | AD AE AF AG AL AM AO AR AT AU AZ BA BB BD BE BF BG BH BI BJ BN BO BR BS BT 2 | BW BY BZ CA CD CF CG CH CI CL CM CN CO CR CU CV CY CZ DE DJ DK DM DZ EC EE 3 | EG ER ES ET FI FJ FM FR GA GB GD GE GH GM GN GQ GR GT GW GY HN HR HT HU ID 4 | IE IL IN IQ IR IS IT JM JO JP KE KG KH KI KM KN KP KR KW KZ LA LB LC LI LK 5 | LR LS LT LU LV LY MA MC MD ME MG MH MK ML MM MN MR MT MU MV MW MX MY MZ NA 6 | NE NG NI NL NO NP NR NZ OM PA PE PG PH PK PL PT PW PY QA RO RS RU RW SA SB 7 | SC SD SE SG SI SK SL SM SN SO SR SS ST SV SY SZ TD TG TH TJ TL TM TN TO TR 8 | TT TV TW TZ UA UG US UY UZ VA VC VE VN VU WS YE ZA ZM ZW 9 | -------------------------------------------------------------------------------- /concorrencia/executors/getflags/flags.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """Download flags of top 20 countries by population 4 | 5 | Sequential version 6 | 7 | Sample runs (first with new domain, so no caching ever):: 8 | 9 | $ ./flags.py 10 | BD BR CD CN DE EG ET FR ID IN IR JP MX NG PH PK RU TR US VN 11 | 20 downloads in 26.21s 12 | $ ./flags.py 13 | BD BR CD CN DE EG ET FR ID IN IR JP MX NG PH PK RU TR US VN 14 | 20 downloads in 14.57s 15 | 16 | 17 | """ 18 | 19 | # tag::FLAGS_PY[] 20 | import time 21 | from pathlib import Path 22 | from typing import Callable 23 | 24 | import httpx # <1> 25 | 26 | POP20_CC = ('IN CN US ID BR PK NG BD RU JP ' 27 | 'MX PH VN ET EG DE IR TR CD FR').split() # <2> 28 | 29 | BASE_URL = 'https://www.fluentpython.com/data/flags' # <3> 30 | DEST_DIR = Path('downloaded') # <4> 31 | 32 | def save_flag(img: bytes, filename: str) -> None: # <5> 33 | (DEST_DIR / filename).write_bytes(img) 34 | 35 | def get_flag(cc: str) -> bytes: # <6> 36 | url = f'{BASE_URL}/{cc}/{cc}.gif'.lower() 37 | resp = httpx.get(url, timeout=6.1, # <7> 38 | follow_redirects=True) # <8> 39 | resp.raise_for_status() # <9> 40 | return resp.content 41 | 42 | def download_many(cc_list: list[str]) -> int: # <10> 43 | for cc in sorted(cc_list): # <11> 44 | image = get_flag(cc) 45 | save_flag(image, f'{cc}.gif') 46 | print(cc, end=' ', flush=True) # <12> 47 | return len(cc_list) 48 | 49 | def main(downloader: Callable[[list[str]], int]) -> None: # <13> 50 | DEST_DIR.mkdir(exist_ok=True) # <14> 51 | t0 = time.perf_counter() # <15> 52 | count = downloader(POP20_CC) 53 | elapsed = time.perf_counter() - t0 54 | print(f'\n{count} downloads in {elapsed:.2f}s') 55 | 56 | if __name__ == '__main__': 57 | main(download_many) # <16> 58 | # end::FLAGS_PY[] 59 | -------------------------------------------------------------------------------- /concorrencia/executors/getflags/flags.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramalho/python-eng/59a007d54595f11441ec7de07909576e1c158772/concorrencia/executors/getflags/flags.zip -------------------------------------------------------------------------------- /concorrencia/executors/getflags/flags2_asyncio.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """Download flags of countries (with error handling). 4 | 5 | asyncio async/await version 6 | 7 | """ 8 | # tag::FLAGS2_ASYNCIO_TOP[] 9 | import asyncio 10 | from collections import Counter 11 | from http import HTTPStatus 12 | from pathlib import Path 13 | 14 | import httpx 15 | import tqdm # type: ignore 16 | 17 | from flags2_common import main, DownloadStatus, save_flag 18 | 19 | # low concurrency default to avoid errors from remote site, 20 | # such as 503 - Service Temporarily Unavailable 21 | DEFAULT_CONCUR_REQ = 5 22 | MAX_CONCUR_REQ = 1000 23 | 24 | async def get_flag(client: httpx.AsyncClient, # <1> 25 | base_url: str, 26 | cc: str) -> bytes: 27 | url = f'{base_url}/{cc}/{cc}.gif'.lower() 28 | resp = await client.get(url, timeout=3.1, follow_redirects=True) # <2> 29 | resp.raise_for_status() 30 | return resp.content 31 | 32 | async def download_one(client: httpx.AsyncClient, 33 | cc: str, 34 | base_url: str, 35 | semaphore: asyncio.Semaphore, 36 | verbose: bool) -> DownloadStatus: 37 | try: 38 | async with semaphore: # <3> 39 | image = await get_flag(client, base_url, cc) 40 | except httpx.HTTPStatusError as exc: # <4> 41 | res = exc.response 42 | if res.status_code == HTTPStatus.NOT_FOUND: 43 | status = DownloadStatus.NOT_FOUND 44 | msg = f'not found: {res.url}' 45 | else: 46 | raise 47 | else: 48 | await asyncio.to_thread(save_flag, image*1000, f'{cc}.gif') # <5> 49 | status = DownloadStatus.OK 50 | msg = 'OK' 51 | if verbose and msg: 52 | print(cc, msg) 53 | return status 54 | # end::FLAGS2_ASYNCIO_TOP[] 55 | 56 | # tag::FLAGS2_ASYNCIO_START[] 57 | async def supervisor(cc_list: list[str], 58 | base_url: str, 59 | verbose: bool, 60 | concur_req: int) -> Counter[DownloadStatus]: # <1> 61 | counter: Counter[DownloadStatus] = Counter() 62 | semaphore = asyncio.Semaphore(concur_req) # <2> 63 | async with httpx.AsyncClient() as client: 64 | to_do = [download_one(client, cc, base_url, semaphore, verbose) 65 | for cc in sorted(cc_list)] # <3> 66 | to_do_iter = asyncio.as_completed(to_do) # <4> 67 | if not verbose: 68 | to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list)) # <5> 69 | error: httpx.HTTPError | None = None # <6> 70 | for coro in to_do_iter: # <7> 71 | try: 72 | status = await coro # <8> 73 | except httpx.HTTPStatusError as exc: 74 | error_msg = 'HTTP error {resp.status_code} - {resp.reason_phrase}' 75 | error_msg = error_msg.format(resp=exc.response) 76 | error = exc # <9> 77 | except httpx.RequestError as exc: 78 | error_msg = f'{exc} {type(exc)}'.strip() 79 | error = exc # <10> 80 | except KeyboardInterrupt: 81 | break 82 | 83 | if error: 84 | status = DownloadStatus.ERROR # <11> 85 | if verbose: 86 | url = str(error.request.url) # <12> 87 | cc = Path(url).stem.upper() # <13> 88 | print(f'{cc} error: {error_msg}') 89 | counter[status] += 1 90 | 91 | return counter 92 | 93 | def download_many(cc_list: list[str], 94 | base_url: str, 95 | verbose: bool, 96 | concur_req: int) -> Counter[DownloadStatus]: 97 | coro = supervisor(cc_list, base_url, verbose, concur_req) 98 | counts = asyncio.run(coro) # <14> 99 | 100 | return counts 101 | 102 | if __name__ == '__main__': 103 | main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ) 104 | # end::FLAGS2_ASYNCIO_START[] 105 | -------------------------------------------------------------------------------- /concorrencia/executors/getflags/flags2_asyncio_executor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """Download flags of countries (with error handling). 4 | 5 | asyncio async/await version 6 | 7 | """ 8 | # tag::FLAGS2_ASYNCIO_TOP[] 9 | import asyncio 10 | from collections import Counter 11 | from http import HTTPStatus 12 | from pathlib import Path 13 | 14 | import httpx 15 | import tqdm # type: ignore 16 | 17 | from flags2_common import main, DownloadStatus, save_flag 18 | 19 | # default set low to avoid errors from remote site, such as 20 | # 503 - Service Temporarily Unavailable 21 | DEFAULT_CONCUR_REQ = 5 22 | MAX_CONCUR_REQ = 1000 23 | 24 | 25 | async def get_flag(client: httpx.AsyncClient, # <2> 26 | base_url: str, 27 | cc: str) -> bytes: 28 | url = f'{base_url}/{cc}/{cc}.gif'.lower() 29 | resp = await client.get(url, timeout=3.1, follow_redirects=True) # <3> 30 | resp.raise_for_status() 31 | return resp.content 32 | 33 | 34 | async def download_one(client: httpx.AsyncClient, 35 | cc: str, 36 | base_url: str, 37 | semaphore: asyncio.Semaphore, 38 | verbose: bool) -> DownloadStatus: 39 | try: 40 | async with semaphore: 41 | image = await get_flag(client, base_url, cc) 42 | except httpx.HTTPStatusError as exc: 43 | res = exc.response 44 | if res.status_code == HTTPStatus.NOT_FOUND: 45 | status = DownloadStatus.NOT_FOUND 46 | msg = f'not found: {res.url}' 47 | else: 48 | raise 49 | else: 50 | # tag::FLAGS2_ASYNCIO_EXECUTOR[] 51 | loop = asyncio.get_running_loop() # <1> 52 | loop.run_in_executor(None, save_flag, # <2> 53 | image, f'{cc}.gif') # <3> 54 | # end::FLAGS2_ASYNCIO_EXECUTOR[] 55 | status = DownloadStatus.OK 56 | msg = 'OK' 57 | if verbose and msg: 58 | print(cc, msg) 59 | return status 60 | 61 | async def supervisor(cc_list: list[str], 62 | base_url: str, 63 | verbose: bool, 64 | concur_req: int) -> Counter[DownloadStatus]: # <1> 65 | counter: Counter[DownloadStatus] = Counter() 66 | semaphore = asyncio.Semaphore(concur_req) # <2> 67 | async with httpx.AsyncClient() as client: 68 | to_do = [download_one(client, cc, base_url, semaphore, verbose) 69 | for cc in sorted(cc_list)] # <3> 70 | to_do_iter = asyncio.as_completed(to_do) # <4> 71 | if not verbose: 72 | to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list)) # <5> 73 | error: httpx.HTTPError | None = None 74 | for coro in to_do_iter: # <6> 75 | try: 76 | status = await coro # <7> 77 | except httpx.HTTPStatusError as exc: # <13> 78 | error_msg = 'HTTP error {resp.status_code} - {resp.reason_phrase}' 79 | error_msg = error_msg.format(resp=exc.response) 80 | error = exc 81 | except httpx.RequestError as exc: # <15> 82 | error_msg = f'{exc} {type(exc)}'.strip() 83 | error = exc 84 | except KeyboardInterrupt: # <7> 85 | break 86 | else: # <8> 87 | error = None 88 | 89 | if error: 90 | status = DownloadStatus.ERROR # <9> 91 | if verbose: # <11> 92 | cc = Path(str(error.request.url)).stem.upper() 93 | print(f'{cc} error: {error_msg}') 94 | counter[status] += 1 # <10> 95 | 96 | return counter # <12> 97 | 98 | def download_many(cc_list: list[str], 99 | base_url: str, 100 | verbose: bool, 101 | concur_req: int) -> Counter[DownloadStatus]: 102 | coro = supervisor(cc_list, base_url, verbose, concur_req) 103 | counts = asyncio.run(coro) # <14> 104 | 105 | return counts 106 | 107 | if __name__ == '__main__': 108 | main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ) 109 | # end::FLAGS2_ASYNCIO_START[] 110 | -------------------------------------------------------------------------------- /concorrencia/executors/getflags/flags2_common.py: -------------------------------------------------------------------------------- 1 | """Utilities for second set of flag examples. 2 | """ 3 | 4 | import argparse 5 | import string 6 | import sys 7 | import time 8 | from collections import Counter 9 | from enum import Enum 10 | from pathlib import Path 11 | 12 | DownloadStatus = Enum('DownloadStatus', 'OK NOT_FOUND ERROR') 13 | 14 | POP20_CC = ('CN IN US ID BR PK NG BD RU JP ' 15 | 'MX PH VN ET EG DE IR TR CD FR').split() 16 | 17 | DEFAULT_CONCUR_REQ = 1 18 | MAX_CONCUR_REQ = 1 19 | 20 | SERVERS = { 21 | 'REMOTE': 'https://www.fluentpython.com/data/flags', 22 | 'LOCAL': 'http://localhost:8000/flags', 23 | 'DELAY': 'http://localhost:8001/flags', 24 | 'ERROR': 'http://localhost:8002/flags', 25 | } 26 | DEFAULT_SERVER = 'LOCAL' 27 | 28 | DEST_DIR = Path('downloaded') 29 | COUNTRY_CODES_FILE = Path('country_codes.txt') 30 | 31 | 32 | def save_flag(img: bytes, filename: str) -> None: 33 | (DEST_DIR / filename).write_bytes(img) 34 | 35 | 36 | def initial_report(cc_list: list[str], 37 | actual_req: int, 38 | server_label: str) -> None: 39 | if len(cc_list) <= 10: 40 | cc_msg = ', '.join(cc_list) 41 | else: 42 | cc_msg = f'from {cc_list[0]} to {cc_list[-1]}' 43 | print(f'{server_label} site: {SERVERS[server_label]}') 44 | plural = 's' if len(cc_list) != 1 else '' 45 | print(f'Searching for {len(cc_list)} flag{plural}: {cc_msg}') 46 | if actual_req == 1: 47 | print('1 connection will be used.') 48 | else: 49 | print(f'{actual_req} concurrent connections will be used.') 50 | 51 | 52 | def final_report(cc_list: list[str], 53 | counter: Counter[DownloadStatus], 54 | start_time: float) -> None: 55 | elapsed = time.perf_counter() - start_time 56 | print('-' * 20) 57 | plural = 's' if counter[DownloadStatus.OK] != 1 else '' 58 | print(f'{counter[DownloadStatus.OK]:3} flag{plural} downloaded.') 59 | if counter[DownloadStatus.NOT_FOUND]: 60 | print(f'{counter[DownloadStatus.NOT_FOUND]:3} not found.') 61 | if counter[DownloadStatus.ERROR]: 62 | plural = 's' if counter[DownloadStatus.ERROR] != 1 else '' 63 | print(f'{counter[DownloadStatus.ERROR]:3} error{plural}.') 64 | print(f'Elapsed time: {elapsed:.2f}s') 65 | 66 | 67 | def expand_cc_args(every_cc: bool, 68 | all_cc: bool, 69 | cc_args: list[str], 70 | limit: int) -> list[str]: 71 | codes: set[str] = set() 72 | A_Z = string.ascii_uppercase 73 | if every_cc: 74 | codes.update(a+b for a in A_Z for b in A_Z) 75 | elif all_cc: 76 | text = COUNTRY_CODES_FILE.read_text() 77 | codes.update(text.split()) 78 | else: 79 | for cc in (c.upper() for c in cc_args): 80 | if len(cc) == 1 and cc in A_Z: 81 | codes.update(cc + c for c in A_Z) 82 | elif len(cc) == 2 and all(c in A_Z for c in cc): 83 | codes.add(cc) 84 | else: 85 | raise ValueError('*** Usage error: each CC argument ' 86 | 'must be A to Z or AA to ZZ.') 87 | return sorted(codes)[:limit] 88 | 89 | 90 | def process_args(default_concur_req): 91 | server_options = ', '.join(sorted(SERVERS)) 92 | parser = argparse.ArgumentParser( 93 | description='Download flags for country codes. ' 94 | 'Default: top 20 countries by population.') 95 | parser.add_argument( 96 | 'cc', metavar='CC', nargs='*', 97 | help='country code or 1st letter (eg. B for BA...BZ)') 98 | parser.add_argument( 99 | '-a', '--all', action='store_true', 100 | help='get all available flags (AD to ZW)') 101 | parser.add_argument( 102 | '-e', '--every', action='store_true', 103 | help='get flags for every possible code (AA...ZZ)') 104 | parser.add_argument( 105 | '-l', '--limit', metavar='N', type=int, help='limit to N first codes', 106 | default=sys.maxsize) 107 | parser.add_argument( 108 | '-m', '--max_req', metavar='CONCURRENT', type=int, 109 | default=default_concur_req, 110 | help=f'maximum concurrent requests (default={default_concur_req})') 111 | parser.add_argument( 112 | '-s', '--server', metavar='LABEL', default=DEFAULT_SERVER, 113 | help=f'Server to hit; one of {server_options} ' 114 | f'(default={DEFAULT_SERVER})') 115 | parser.add_argument( 116 | '-v', '--verbose', action='store_true', 117 | help='output detailed progress info') 118 | args = parser.parse_args() 119 | if args.max_req < 1: 120 | print('*** Usage error: --max_req CONCURRENT must be >= 1') 121 | parser.print_usage() 122 | # "standard" exit status codes: 123 | # https://stackoverflow.com/questions/1101957/are-there-any-standard-exit-status-codes-in-linux/40484670#40484670 124 | sys.exit(2) # command line usage error 125 | if args.limit < 1: 126 | print('*** Usage error: --limit N must be >= 1') 127 | parser.print_usage() 128 | sys.exit(2) # command line usage error 129 | args.server = args.server.upper() 130 | if args.server not in SERVERS: 131 | print(f'*** Usage error: --server LABEL ' 132 | f'must be one of {server_options}') 133 | parser.print_usage() 134 | sys.exit(2) # command line usage error 135 | try: 136 | cc_list = expand_cc_args(args.every, args.all, args.cc, args.limit) 137 | except ValueError as exc: 138 | print(exc.args[0]) 139 | parser.print_usage() 140 | sys.exit(2) # command line usage error 141 | 142 | if not cc_list: 143 | cc_list = sorted(POP20_CC)[:args.limit] 144 | return args, cc_list 145 | 146 | 147 | def main(download_many, default_concur_req, max_concur_req): 148 | args, cc_list = process_args(default_concur_req) 149 | actual_req = min(args.max_req, max_concur_req, len(cc_list)) 150 | initial_report(cc_list, actual_req, args.server) 151 | base_url = SERVERS[args.server] 152 | DEST_DIR.mkdir(exist_ok=True) 153 | t0 = time.perf_counter() 154 | counter = download_many(cc_list, base_url, args.verbose, actual_req) 155 | final_report(cc_list, counter, t0) 156 | -------------------------------------------------------------------------------- /concorrencia/executors/getflags/flags2_sequential.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """Download flags of countries (with error handling). 4 | 5 | Sequential version 6 | 7 | Sample run:: 8 | 9 | $ python3 flags2_sequential.py -s DELAY b 10 | DELAY site: http://localhost:8002/flags 11 | Searching for 26 flags: from BA to BZ 12 | 1 concurrent connection will be used. 13 | -------------------- 14 | 17 flags downloaded. 15 | 9 not found. 16 | Elapsed time: 13.36s 17 | 18 | """ 19 | 20 | # tag::FLAGS2_BASIC_HTTP_FUNCTIONS[] 21 | from collections import Counter 22 | from http import HTTPStatus 23 | 24 | import httpx 25 | import tqdm # type: ignore # <1> 26 | 27 | from flags2_common import main, save_flag, DownloadStatus # <2> 28 | 29 | DEFAULT_CONCUR_REQ = 1 30 | MAX_CONCUR_REQ = 1 31 | 32 | def get_flag(base_url: str, cc: str) -> bytes: 33 | url = f'{base_url}/{cc}/{cc}.gif'.lower() 34 | resp = httpx.get(url, timeout=3.1, follow_redirects=True) 35 | resp.raise_for_status() # <3> 36 | return resp.content 37 | 38 | def download_one(cc: str, base_url: str, verbose: bool = False) -> DownloadStatus: 39 | try: 40 | image = get_flag(base_url, cc) 41 | except httpx.HTTPStatusError as exc: # <4> 42 | res = exc.response 43 | if res.status_code == HTTPStatus.NOT_FOUND: 44 | status = DownloadStatus.NOT_FOUND # <5> 45 | msg = f'not found: {res.url}' 46 | else: 47 | raise # <6> 48 | else: 49 | save_flag(image, f'{cc}.gif') 50 | status = DownloadStatus.OK 51 | msg = 'OK' 52 | 53 | if verbose: # <7> 54 | print(cc, msg) 55 | 56 | return status 57 | # end::FLAGS2_BASIC_HTTP_FUNCTIONS[] 58 | 59 | # tag::FLAGS2_DOWNLOAD_MANY_SEQUENTIAL[] 60 | def download_many(cc_list: list[str], 61 | base_url: str, 62 | verbose: bool, 63 | _unused_concur_req: int) -> Counter[DownloadStatus]: 64 | counter: Counter[DownloadStatus] = Counter() # <1> 65 | cc_iter = sorted(cc_list) # <2> 66 | if not verbose: 67 | cc_iter = tqdm.tqdm(cc_iter) # <3> 68 | for cc in cc_iter: 69 | try: 70 | status = download_one(cc, base_url, verbose) # <4> 71 | except httpx.HTTPStatusError as exc: # <5> 72 | error_msg = 'HTTP error {resp.status_code} - {resp.reason_phrase}' 73 | error_msg = error_msg.format(resp=exc.response) 74 | except httpx.RequestError as exc: # <6> 75 | error_msg = f'{exc} {type(exc)}'.strip() 76 | except KeyboardInterrupt: # <7> 77 | break 78 | else: # <8> 79 | error_msg = '' 80 | 81 | if error_msg: 82 | status = DownloadStatus.ERROR # <9> 83 | counter[status] += 1 # <10> 84 | if verbose and error_msg: # <11> 85 | print(f'{cc} error: {error_msg}') 86 | 87 | return counter # <12> 88 | # end::FLAGS2_DOWNLOAD_MANY_SEQUENTIAL[] 89 | 90 | if __name__ == '__main__': 91 | main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ) 92 | -------------------------------------------------------------------------------- /concorrencia/executors/getflags/flags2_threadpool.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """Download flags of countries (with error handling). 4 | 5 | ThreadPool version 6 | 7 | Sample run:: 8 | 9 | $ python3 flags2_threadpool.py -s ERROR -e 10 | ERROR site: http://localhost:8003/flags 11 | Searching for 676 flags: from AA to ZZ 12 | 30 concurrent connections will be used. 13 | -------------------- 14 | 150 flags downloaded. 15 | 361 not found. 16 | 165 errors. 17 | Elapsed time: 7.46s 18 | 19 | """ 20 | 21 | # tag::FLAGS2_THREADPOOL[] 22 | from collections import Counter 23 | from concurrent.futures import ThreadPoolExecutor, as_completed 24 | 25 | import httpx 26 | import tqdm # type: ignore 27 | 28 | from flags2_common import main, DownloadStatus 29 | from flags2_sequential import download_one # <1> 30 | 31 | DEFAULT_CONCUR_REQ = 30 # <2> 32 | MAX_CONCUR_REQ = 1000 # <3> 33 | 34 | 35 | def download_many(cc_list: list[str], 36 | base_url: str, 37 | verbose: bool, 38 | concur_req: int) -> Counter[DownloadStatus]: 39 | counter: Counter[DownloadStatus] = Counter() 40 | with ThreadPoolExecutor(max_workers=concur_req) as executor: # <4> 41 | to_do_map = {} # <5> 42 | for cc in sorted(cc_list): # <6> 43 | future = executor.submit(download_one, cc, 44 | base_url, verbose) # <7> 45 | to_do_map[future] = cc # <8> 46 | done_iter = as_completed(to_do_map) # <9> 47 | if not verbose: 48 | done_iter = tqdm.tqdm(done_iter, total=len(cc_list)) # <10> 49 | for future in done_iter: # <11> 50 | try: 51 | status = future.result() # <12> 52 | except httpx.HTTPStatusError as exc: # <13> 53 | error_msg = 'HTTP error {resp.status_code} - {resp.reason_phrase}' 54 | error_msg = error_msg.format(resp=exc.response) 55 | except httpx.RequestError as exc: 56 | error_msg = f'{exc} {type(exc)}'.strip() 57 | except KeyboardInterrupt: 58 | break 59 | else: 60 | error_msg = '' 61 | 62 | if error_msg: 63 | status = DownloadStatus.ERROR 64 | counter[status] += 1 65 | if verbose and error_msg: 66 | cc = to_do_map[future] # <14> 67 | print(f'{cc} error: {error_msg}') 68 | 69 | return counter 70 | 71 | 72 | if __name__ == '__main__': 73 | main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ) 74 | # end::FLAGS2_THREADPOOL[] 75 | -------------------------------------------------------------------------------- /concorrencia/executors/getflags/flags3_asyncio.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """Download flags of countries (with error handling). 4 | 5 | asyncio async/await version 6 | 7 | """ 8 | # tag::FLAGS2_ASYNCIO_TOP[] 9 | import asyncio 10 | from collections import Counter 11 | from http import HTTPStatus 12 | from pathlib import Path 13 | 14 | import httpx 15 | import tqdm # type: ignore 16 | 17 | from flags2_common import main, DownloadStatus, save_flag 18 | 19 | # low concurrency default to avoid errors from remote site, 20 | # such as 503 - Service Temporarily Unavailable 21 | DEFAULT_CONCUR_REQ = 5 22 | MAX_CONCUR_REQ = 1000 23 | 24 | async def get_flag(client: httpx.AsyncClient, # <1> 25 | base_url: str, 26 | cc: str) -> bytes: 27 | url = f'{base_url}/{cc}/{cc}.gif'.lower() 28 | resp = await client.get(url, timeout=3.1, follow_redirects=True) # <2> 29 | resp.raise_for_status() 30 | return resp.content 31 | 32 | # tag::FLAGS3_ASYNCIO_GET_COUNTRY[] 33 | async def get_country(client: httpx.AsyncClient, 34 | base_url: str, 35 | cc: str) -> str: # <1> 36 | url = f'{base_url}/{cc}/metadata.json'.lower() 37 | resp = await client.get(url, timeout=3.1, follow_redirects=True) 38 | resp.raise_for_status() 39 | metadata = resp.json() # <2> 40 | return metadata['country'] # <3> 41 | # end::FLAGS3_ASYNCIO_GET_COUNTRY[] 42 | 43 | # tag::FLAGS3_ASYNCIO_DOWNLOAD_ONE[] 44 | async def download_one(client: httpx.AsyncClient, 45 | cc: str, 46 | base_url: str, 47 | semaphore: asyncio.Semaphore, 48 | verbose: bool) -> DownloadStatus: 49 | try: 50 | async with semaphore: # <1> 51 | image = await get_flag(client, base_url, cc) 52 | async with semaphore: # <2> 53 | country = await get_country(client, base_url, cc) 54 | except httpx.HTTPStatusError as exc: 55 | res = exc.response 56 | if res.status_code == HTTPStatus.NOT_FOUND: 57 | status = DownloadStatus.NOT_FOUND 58 | msg = f'not found: {res.url}' 59 | else: 60 | raise 61 | else: 62 | filename = country.replace(' ', '_') # <3> 63 | await asyncio.to_thread(save_flag, image, f'{filename}.gif') 64 | status = DownloadStatus.OK 65 | msg = 'OK' 66 | if verbose and msg: 67 | print(cc, msg) 68 | return status 69 | # end::FLAGS3_ASYNCIO_DOWNLOAD_ONE[] 70 | 71 | # tag::FLAGS2_ASYNCIO_START[] 72 | async def supervisor(cc_list: list[str], 73 | base_url: str, 74 | verbose: bool, 75 | concur_req: int) -> Counter[DownloadStatus]: # <1> 76 | counter: Counter[DownloadStatus] = Counter() 77 | semaphore = asyncio.Semaphore(concur_req) # <2> 78 | async with httpx.AsyncClient() as client: 79 | to_do = [download_one(client, cc, base_url, semaphore, verbose) 80 | for cc in sorted(cc_list)] # <3> 81 | to_do_iter = asyncio.as_completed(to_do) # <4> 82 | if not verbose: 83 | to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list)) # <5> 84 | error: httpx.HTTPError | None = None # <6> 85 | for coro in to_do_iter: # <7> 86 | try: 87 | status = await coro # <8> 88 | except httpx.HTTPStatusError as exc: 89 | error_msg = 'HTTP error {resp.status_code} - {resp.reason_phrase}' 90 | error_msg = error_msg.format(resp=exc.response) 91 | error = exc # <9> 92 | except httpx.RequestError as exc: 93 | error_msg = f'{exc} {type(exc)}'.strip() 94 | error = exc # <10> 95 | except KeyboardInterrupt: 96 | break 97 | 98 | if error: 99 | status = DownloadStatus.ERROR # <11> 100 | if verbose: 101 | url = str(error.request.url) # <12> 102 | cc = Path(url).stem.upper() # <13> 103 | print(f'{cc} error: {error_msg}') 104 | counter[status] += 1 105 | 106 | return counter 107 | 108 | def download_many(cc_list: list[str], 109 | base_url: str, 110 | verbose: bool, 111 | concur_req: int) -> Counter[DownloadStatus]: 112 | coro = supervisor(cc_list, base_url, verbose, concur_req) 113 | counts = asyncio.run(coro) # <14> 114 | 115 | return counts 116 | 117 | if __name__ == '__main__': 118 | main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ) 119 | # end::FLAGS2_ASYNCIO_START[] 120 | -------------------------------------------------------------------------------- /concorrencia/executors/getflags/flags_asyncio.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """Download flags of top 20 countries by population 4 | 5 | asyncio + aiottp version 6 | 7 | Sample run:: 8 | 9 | $ python3 flags_asyncio.py 10 | EG VN IN TR RU ID US DE CN MX JP BD NG ET FR BR PH PK CD IR 11 | 20 flags downloaded in 1.07s 12 | """ 13 | # tag::FLAGS_ASYNCIO_TOP[] 14 | import asyncio 15 | 16 | from httpx import AsyncClient # <1> 17 | 18 | from flags import BASE_URL, save_flag, main # <2> 19 | 20 | async def download_one(client: AsyncClient, cc: str): # <3> 21 | image = await get_flag(client, cc) 22 | save_flag(image, f'{cc}.gif') 23 | print(cc, end=' ', flush=True) 24 | return cc 25 | 26 | async def get_flag(client: AsyncClient, cc: str) -> bytes: # <4> 27 | url = f'{BASE_URL}/{cc}/{cc}.gif'.lower() 28 | resp = await client.get(url, timeout=6.1, 29 | follow_redirects=True) # <5> 30 | return resp.read() # <6> 31 | # end::FLAGS_ASYNCIO_TOP[] 32 | 33 | # tag::FLAGS_ASYNCIO_START[] 34 | def download_many(cc_list: list[str]) -> int: # <1> 35 | return asyncio.run(supervisor(cc_list)) # <2> 36 | 37 | async def supervisor(cc_list: list[str]) -> int: 38 | async with AsyncClient() as client: # <3> 39 | to_do = [download_one(client, cc) 40 | for cc in sorted(cc_list)] # <4> 41 | res = await asyncio.gather(*to_do) # <5> 42 | 43 | return len(res) # <6> 44 | 45 | if __name__ == '__main__': 46 | main(download_many) 47 | # end::FLAGS_ASYNCIO_START[] 48 | -------------------------------------------------------------------------------- /concorrencia/executors/getflags/flags_threadpool.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """Download flags of top 20 countries by population 4 | 5 | ThreadPoolExecutor version 6 | 7 | Sample run:: 8 | 9 | $ python3 flags_threadpool.py 10 | DE FR BD CN EG RU IN TR VN ID JP BR NG MX PK ET PH CD US IR 11 | 20 downloads in 0.35s 12 | 13 | """ 14 | 15 | # tag::FLAGS_THREADPOOL[] 16 | from concurrent import futures 17 | 18 | from flags import save_flag, get_flag, main # <1> 19 | 20 | def download_one(cc: str): # <2> 21 | image = get_flag(cc) 22 | save_flag(image, f'{cc}.gif') 23 | print(cc, end=' ', flush=True) 24 | return cc 25 | 26 | def download_many(cc_list: list[str]) -> int: 27 | with futures.ThreadPoolExecutor() as executor: # <3> 28 | res = executor.map(download_one, sorted(cc_list)) # <4> 29 | 30 | return len(list(res)) # <5> 31 | 32 | if __name__ == '__main__': 33 | main(download_many) # <6> 34 | # end::FLAGS_THREADPOOL[] 35 | -------------------------------------------------------------------------------- /concorrencia/executors/getflags/flags_threadpool_futures.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """Download flags of top 20 countries by population 4 | 5 | ThreadPoolExecutor example with ``as_completed``. 6 | """ 7 | from concurrent import futures 8 | 9 | from flags import main 10 | from flags_threadpool import download_one 11 | 12 | 13 | # tag::FLAGS_THREADPOOL_AS_COMPLETED[] 14 | def download_many(cc_list: list[str]) -> int: 15 | cc_list = cc_list[:5] # <1> 16 | with futures.ThreadPoolExecutor(max_workers=3) as executor: # <2> 17 | to_do: list[futures.Future] = [] 18 | for cc in sorted(cc_list): # <3> 19 | future = executor.submit(download_one, cc) # <4> 20 | to_do.append(future) # <5> 21 | print(f'Scheduled for {cc}: {future}') # <6> 22 | 23 | for count, future in enumerate(futures.as_completed(to_do), 1): # <7> 24 | res: str = future.result() # <8> 25 | print(f'{future} result: {res!r}') # <9> 26 | 27 | return count 28 | # end::FLAGS_THREADPOOL_AS_COMPLETED[] 29 | 30 | if __name__ == '__main__': 31 | main(download_many) 32 | -------------------------------------------------------------------------------- /concorrencia/executors/getflags/httpx-error-tree/drawtree.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from tree import tree 4 | 5 | 6 | SP = '\N{SPACE}' 7 | HLIN = '\N{BOX DRAWINGS LIGHT HORIZONTAL}' # ─ 8 | ELBOW = f'\N{BOX DRAWINGS LIGHT UP AND RIGHT}{HLIN*2}{SP}' # └── 9 | TEE = f'\N{BOX DRAWINGS LIGHT VERTICAL AND RIGHT}{HLIN*2}{SP}' # ├── 10 | PIPE = f'\N{BOX DRAWINGS LIGHT VERTICAL}{SP*3}' # │ 11 | 12 | 13 | def cls_name(cls): 14 | module = 'builtins.' if cls.__module__ == 'builtins' else '' 15 | return module + cls.__name__ 16 | 17 | def render_lines(tree_iter): 18 | cls, _, _ = next(tree_iter) 19 | yield cls_name(cls) 20 | prefix = '' 21 | 22 | for cls, level, last in tree_iter: 23 | prefix = prefix[:4 * (level-1)] 24 | prefix = prefix.replace(TEE, PIPE).replace(ELBOW, SP*4) 25 | prefix += ELBOW if last else TEE 26 | yield prefix + cls_name(cls) 27 | 28 | 29 | def draw(cls): 30 | for line in render_lines(tree(cls)): 31 | print(line) 32 | 33 | 34 | if __name__ == '__main__': 35 | draw(Exception) 36 | -------------------------------------------------------------------------------- /concorrencia/executors/getflags/httpx-error-tree/tree.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import httpx # make httpx classes available to .__subclasses__() 4 | 5 | 6 | def tree(cls, level=0, last_sibling=True): 7 | yield cls, level, last_sibling 8 | 9 | # get RuntimeError and exceptions defined in httpx 10 | subclasses = [sub for sub in cls.__subclasses__() 11 | if sub is RuntimeError or sub.__module__ == 'httpx'] 12 | if subclasses: 13 | last = subclasses[-1] 14 | for sub in subclasses: 15 | yield from tree(sub, level+1, sub is last) 16 | 17 | 18 | def display(cls): 19 | for cls, level, _ in tree(cls): 20 | indent = ' ' * 4 * level 21 | module = 'builtins.' if cls.__module__ == 'builtins' else '' 22 | print(f'{indent}{module}{cls.__name__}') 23 | 24 | 25 | if __name__ == '__main__': 26 | display(Exception) 27 | -------------------------------------------------------------------------------- /concorrencia/executors/getflags/requirements.txt: -------------------------------------------------------------------------------- 1 | anyio==3.7.0 2 | certifi==2023.5.7 3 | h11==0.14.0 4 | httpcore==0.17.2 5 | httpx==0.24.1 6 | idna==3.4 7 | sniffio==1.3.0 8 | tqdm==4.65.0 9 | -------------------------------------------------------------------------------- /concorrencia/executors/getflags/slow_server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """Slow HTTP server class. 4 | 5 | This module implements a ThreadingHTTPServer using a custom 6 | SimpleHTTPRequestHandler subclass that introduces delays to all 7 | GET responses, and optionally returns errors to a fraction of 8 | the requests if given the --error_rate command-line argument. 9 | """ 10 | 11 | import contextlib 12 | import os 13 | import socket 14 | import time 15 | from functools import partial 16 | from http import server, HTTPStatus 17 | from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler 18 | from random import random, uniform 19 | 20 | MIN_DELAY = 0.5 # minimum delay for do_GET (seconds) 21 | MAX_DELAY = 5.0 # maximum delay for do_GET (seconds) 22 | 23 | class SlowHTTPRequestHandler(SimpleHTTPRequestHandler): 24 | """SlowHTTPRequestHandler adds delays and errors to test HTTP clients. 25 | 26 | The optional error_rate argument determines how often GET requests 27 | receive a 418 status code, "I'm a teapot". 28 | If error_rate is .15, there's a 15% probability of each GET request 29 | getting that error. 30 | When the server believes it is a teapot, it refuses requests to serve files. 31 | 32 | See: https://tools.ietf.org/html/rfc2324#section-2.3.2 33 | """ 34 | 35 | def __init__(self, *args, error_rate=0.0, **kwargs): 36 | self.error_rate = error_rate 37 | super().__init__(*args, **kwargs) 38 | 39 | def do_GET(self): 40 | """Serve a GET request.""" 41 | delay = uniform(MIN_DELAY, MAX_DELAY) 42 | cc = self.path[-6:-4].upper() 43 | print(f'{cc} delay: {delay:0.2}s') 44 | time.sleep(delay) 45 | if random() < self.error_rate: 46 | # HTTPStatus.IM_A_TEAPOT requires Python >= 3.9 47 | try: 48 | self.send_error(HTTPStatus.IM_A_TEAPOT, "I'm a Teapot") 49 | except BrokenPipeError as exc: 50 | print(f'{cc} *** BrokenPipeError: client closed') 51 | else: 52 | f = self.send_head() 53 | if f: 54 | try: 55 | self.copyfile(f, self.wfile) 56 | except BrokenPipeError as exc: 57 | print(f'{cc} *** BrokenPipeError: client closed') 58 | finally: 59 | f.close() 60 | 61 | # The code in the `if` block below, including comments, was copied 62 | # and adapted from the `http.server` module of Python 3.9 63 | # https://github.com/python/cpython/blob/master/Lib/http/server.py 64 | 65 | if __name__ == '__main__': 66 | import argparse 67 | 68 | parser = argparse.ArgumentParser() 69 | parser.add_argument('--bind', '-b', metavar='ADDRESS', 70 | help='Specify alternate bind address ' 71 | '[default: all interfaces]') 72 | parser.add_argument('--directory', '-d', default=os.getcwd(), 73 | help='Specify alternative directory ' 74 | '[default:current directory]') 75 | parser.add_argument('--error-rate', '-e', metavar='PROBABILITY', 76 | default=0.0, type=float, 77 | help='Error rate; e.g. use .25 for 25%% probability ' 78 | '[default:0.0]') 79 | parser.add_argument('port', action='store', 80 | default=8001, type=int, 81 | nargs='?', 82 | help='Specify alternate port [default: 8001]') 83 | args = parser.parse_args() 84 | handler_class = partial(SlowHTTPRequestHandler, 85 | directory=args.directory, 86 | error_rate=args.error_rate) 87 | 88 | # ensure dual-stack is not disabled; ref #38907 89 | class DualStackServer(ThreadingHTTPServer): 90 | def server_bind(self): 91 | # suppress exception when protocol is IPv4 92 | with contextlib.suppress(Exception): 93 | self.socket.setsockopt( 94 | socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) 95 | return super().server_bind() 96 | 97 | # test is a top-level function in http.server omitted from __all__ 98 | server.test( # type: ignore 99 | HandlerClass=handler_class, 100 | ServerClass=DualStackServer, 101 | port=args.port, 102 | bind=args.bind, 103 | ) 104 | -------------------------------------------------------------------------------- /concorrencia/executors/primes/primes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import math 4 | 5 | 6 | PRIME_FIXTURE = [ 7 | (2, True), 8 | (142702110479723, True), 9 | (299593572317531, True), 10 | (3333333333333301, True), 11 | (3333333333333333, False), 12 | (3333335652092209, False), 13 | (4444444444444423, True), 14 | (4444444444444444, False), 15 | (4444444488888889, False), 16 | (5555553133149889, False), 17 | (5555555555555503, True), 18 | (5555555555555555, False), 19 | (6666666666666666, False), 20 | (6666666666666719, True), 21 | (6666667141414921, False), 22 | (7777777536340681, False), 23 | (7777777777777753, True), 24 | (7777777777777777, False), 25 | (9999999999999917, True), 26 | (9999999999999999, False), 27 | ] 28 | 29 | NUMBERS = [n for n, _ in PRIME_FIXTURE] 30 | 31 | # tag::IS_PRIME[] 32 | def is_prime(n: int) -> bool: 33 | if n < 2: 34 | return False 35 | if n == 2: 36 | return True 37 | if n % 2 == 0: 38 | return False 39 | 40 | root = math.isqrt(n) 41 | for i in range(3, root + 1, 2): 42 | if n % i == 0: 43 | return False 44 | return True 45 | # end::IS_PRIME[] 46 | 47 | if __name__ == '__main__': 48 | 49 | for n, prime in PRIME_FIXTURE: 50 | prime_res = is_prime(n) 51 | assert prime_res == prime 52 | print(n, prime) 53 | -------------------------------------------------------------------------------- /concorrencia/executors/primes/proc_pool.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | proc_pool.py: a version of the proc.py example from chapter 20, 5 | but using `concurrent.futures.ProcessPoolExecutor`. 6 | """ 7 | 8 | # tag::PRIMES_POOL[] 9 | import sys 10 | from concurrent import futures # <1> 11 | from time import perf_counter 12 | from typing import NamedTuple 13 | 14 | from primes import is_prime, NUMBERS 15 | 16 | class PrimeResult(NamedTuple): # <2> 17 | n: int 18 | flag: bool 19 | elapsed: float 20 | 21 | def check(n: int) -> PrimeResult: 22 | t0 = perf_counter() 23 | res = is_prime(n) 24 | return PrimeResult(n, res, perf_counter() - t0) 25 | 26 | def main() -> None: 27 | if len(sys.argv) < 2: 28 | workers = None # <3> 29 | else: 30 | workers = int(sys.argv[1]) 31 | 32 | executor = futures.ProcessPoolExecutor(workers) # <4> 33 | actual_workers = executor._max_workers # type: ignore # <5> 34 | 35 | print(f'Checking {len(NUMBERS)} numbers with {actual_workers} processes:') 36 | 37 | t0 = perf_counter() 38 | 39 | numbers = sorted(NUMBERS, reverse=True) # <6> 40 | with executor: # <7> 41 | for n, prime, elapsed in executor.map(check, numbers): # <8> 42 | label = 'P' if prime else ' ' 43 | print(f'{n:16} {label} {elapsed:9.6f}s') 44 | 45 | time = perf_counter() - t0 46 | print(f'Total time: {time:.2f}s') 47 | 48 | if __name__ == '__main__': 49 | main() 50 | # end::PRIMES_POOL[] 51 | -------------------------------------------------------------------------------- /concorrencia/fio.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import time 3 | from threading import Thread, Event 4 | 5 | from primes import is_prime 6 | 7 | def girar(msg: str, pronto: Event) -> None: 8 | for char in itertools.cycle(r'\|/-'): 9 | status = f'\r{char} {msg}' 10 | print(status, end='', flush=True) 11 | if pronto.wait(.05): 12 | break 13 | blanks = ' ' * len(status) 14 | print(f'\r{blanks}\r', end='') 15 | 16 | 17 | def supervisor(): 18 | pronto = Event() 19 | girador = Thread(target=girar, args=['Computando...', pronto]) 20 | girador.start() 21 | n = 7_777_777_777_777_753 22 | primo = is_prime(n) 23 | pronto.set() 24 | girador.join() 25 | e_nao_e = 'é' if primo else 'não é' 26 | print(n, e_nao_e, 'primo' ) 27 | 28 | 29 | 30 | if __name__ == '__main__': 31 | supervisor() 32 | -------------------------------------------------------------------------------- /concorrencia/giro.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import time 3 | 4 | def girar(msg: str) -> None: 5 | for char in itertools.cycle(r'\|/-'): 6 | status = f'\r{char} {msg}' 7 | print(status, end='', flush=True) 8 | time.sleep(.05) 9 | blanks = ' ' * len(status) 10 | print(f'\r{blanks}\r', end='') 11 | 12 | 13 | if __name__ == '__main__': 14 | try: 15 | girar('pensando...') 16 | except KeyboardInterrupt: 17 | pass 18 | -------------------------------------------------------------------------------- /concorrencia/modelos-de-concorrencia.odp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramalho/python-eng/59a007d54595f11441ec7de07909576e1c158772/concorrencia/modelos-de-concorrencia.odp -------------------------------------------------------------------------------- /concorrencia/modelos-de-concorrencia.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramalho/python-eng/59a007d54595f11441ec7de07909576e1c158772/concorrencia/modelos-de-concorrencia.pdf -------------------------------------------------------------------------------- /concorrencia/primes/.gitignore: -------------------------------------------------------------------------------- 1 | primes 2 | -------------------------------------------------------------------------------- /concorrencia/primes/1_primo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import time 4 | 5 | from primes import is_prime 6 | 7 | 8 | def main(): 9 | n = 18_014_398_509_481_951 10 | t0 = time.perf_counter() 11 | primo = is_prime(n) 12 | e_nao_e = 'é' if primo else 'não é' 13 | elapsed = time.perf_counter() - t0 14 | print(f'{n:_d} {e_nao_e} primo ({elapsed:3.1f}s)') 15 | 16 | 17 | if __name__ == '__main__': 18 | main() 19 | -------------------------------------------------------------------------------- /concorrencia/primes/1_primo_async_NO_SPIN.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # credits: Example by Luciano Ramalho inspired by 4 | # Michele Simionato's multiprocessing example in the python-list: 5 | # https://mail.python.org/pipermail/python-list/2009-February/675659.html 6 | 7 | import asyncio 8 | import itertools 9 | import time 10 | 11 | from primes import is_prime 12 | 13 | async def spin(msg: str) -> None: 14 | for char in itertools.cycle(r'\|/-'): 15 | status = f'\r{char} {msg}' 16 | print(status, flush=True, end='') 17 | try: 18 | await asyncio.sleep(.1) 19 | except asyncio.CancelledError: 20 | break 21 | print('THIS WILL NEVER BE OUTPUT') 22 | 23 | 24 | async def supervisor(n: int) -> int: 25 | spinner = asyncio.create_task(spin('thinking!')) # <1> 26 | print(f'spinner object: {spinner}') # <2> 27 | result = is_prime(n) 28 | spinner.cancel() # <5> 29 | return result 30 | 31 | 32 | def main() -> None: 33 | n = 432_345_564_227_567_561 34 | result = asyncio.run(supervisor(n)) 35 | e_nao_e = 'é' if result else 'não é' 36 | print(n, e_nao_e, 'primo') 37 | 38 | if __name__ == '__main__': 39 | main() 40 | -------------------------------------------------------------------------------- /concorrencia/primes/1_primo_proc_spin.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import itertools 4 | from multiprocessing import Process, Event 5 | from multiprocessing.synchronize import Event as EventType 6 | 7 | from primes import is_prime 8 | 9 | 10 | def girar(msg: str, pronto: EventType) -> None: 11 | for char in itertools.cycle(r'\|/-'): 12 | status = f'\r{char} {msg}' 13 | print(status, end='', flush=True) 14 | if pronto.wait(0.05): 15 | break 16 | blanks = ' ' * len(status) 17 | print(f'\r{blanks}\r', end='') 18 | 19 | 20 | def main(): 21 | pronto = Event() 22 | girador = Process(target=girar, args=['Computando...', pronto]) 23 | girador.start() 24 | n = 432_345_564_227_567_561 25 | primo = is_prime(n) 26 | pronto.set() 27 | girador.join() 28 | e_nao_e = 'é' if primo else 'não é' 29 | print(n, e_nao_e, 'primo') 30 | 31 | 32 | if __name__ == '__main__': 33 | main() 34 | -------------------------------------------------------------------------------- /concorrencia/primes/1_primo_thread_spin.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import itertools 4 | from threading import Thread, Event 5 | 6 | from primes import is_prime 7 | 8 | 9 | def girar(msg: str, pronto: Event) -> None: 10 | for char in itertools.cycle(r'\|/-'): 11 | status = f'\r{char} {msg}' 12 | print(status, end='', flush=True) 13 | if pronto.wait(0.05): 14 | break 15 | blanks = ' ' * len(status) 16 | print(f'\r{blanks}\r', end='') 17 | 18 | 19 | def main(): 20 | pronto = Event() 21 | girador = Thread(target=girar, args=['Computando...', pronto]) 22 | girador.start() 23 | n = 432_345_564_227_567_561 24 | primo = is_prime(n) 25 | pronto.set() 26 | girador.join() 27 | e_nao_e = 'é' if primo else 'não é' 28 | print(n, e_nao_e, 'primo') 29 | 30 | 31 | if __name__ == '__main__': 32 | main() 33 | -------------------------------------------------------------------------------- /concorrencia/primes/README.md: -------------------------------------------------------------------------------- 1 | # Notes 2 | 3 | ## CPU usage 4 | 5 | How to get CPU usage in multi-process programs: 6 | 7 | https://stackoverflow.com/questions/276281/cpu-usage-per-process-in-python 8 | 9 | ## Concurrent benchmarks 10 | 11 | ### Interpreter versions 12 | 13 | ``` 14 | $ python3 --version 15 | Python 3.10.12 16 | $ pypy3 --version 17 | Python 3.8.13 (7.3.9+dfsg-1, Apr 01 2022, 21:41:47) 18 | [PyPy 7.3.9 with GCC 11.2.0] 19 | ``` 20 | 21 | ### Sample run 22 | 23 | Script compatible with Python 3.8 (supported by PyPy 7.3.9) 24 | 25 | ``` 26 | $ pypy3 n_primos_proc_py38.py 27 | Checking 20 numbers with 16 processes: 28 | 2 P 0.000004s 29 | 3333333333333333 0.000004s 30 | 4444444444444444 0.000006s 31 | 5555555555555555 0.000046s 32 | 6666666666666666 0.000012s 33 | 7777777777777777 0.000044s 34 | 9999999999999999 0.000005s 35 | 142702110479723 P 0.015781s 36 | 299593572317531 P 0.020436s 37 | 4444444488888889 0.063057s 38 | 5555555555555503 P 0.065277s 39 | 3333333333333301 P 0.083176s 40 | 3333335652092209 0.085459s 41 | 7777777777777753 P 0.090459s 42 | 4444444444444423 P 0.100379s 43 | 6666667141414921 0.099932s 44 | 5555553133149889 0.111482s 45 | 6666666666666719 P 0.116083s 46 | 7777777536340681 0.116376s 47 | 9999999999999917 P 0.129154s 48 | 20 checks in 0.14s 49 | ``` 50 | 51 | ### Python 3.10 v. PyPy3 3.8 [7.3] 52 | 53 | Running on Ubuntu 22.04 (WSL 2, Windows 11), Intel Core i9. 54 | 55 | ``` 56 | $ python3 n_primes_proc_py38.py | (head -1 && tail -1) 57 | Checking 20 numbers with 16 processes: 58 | 20 checks in 3.18s 59 | $ pypy3 n_primes_proc_py38.py | (head -1 && tail -1) 60 | Checking 20 numbers with 16 processes: 61 | 20 checks in 0.16s 62 | ``` 63 | 64 | * This Core i9 has 16 logical cores (8 actual cores × 2 thanks to Hyperthreading™). 65 | * Pypy3 is much faster than CPython for this task. 66 | 67 | Running on MacOS 13.5, Apple M2 Max: 68 | 69 | ``` 70 | % python3.11 n_primes_proc.py | (head -1 && tail -1) 71 | Checking 20 numbers with 12 processes: 72 | 20 checks in 1.12s 73 | % pypy3 n_primes_proc.py | (head -1 && tail -1) 74 | Checking 20 numbers with 12 processes: 75 | 20 checks in 0.15s 76 | ``` 77 | 78 | * This M2 Max has 12 cores. 79 | * CPython is almost 3x faster on M2 Max than on Core i9. 80 | * The performance of Pypy is nearly the same on M2 Max and Core i9. 81 | 82 | ### Python 3.9 v. Python 3.11 83 | 84 | Running on Raspberri Pi 4 (8GB). 85 | 86 | ``` 87 | $ python3.9 n_primes_proc.py | (head -1 && tail -1) 88 | Checking 20 numbers with 4 processes: 89 | 20 checks in 37.09s 90 | $ python3.11 n_primes_proc.py | (head -1 && tail -1) 91 | Checking 20 numbers with 4 processes: 92 | 20 checks in 24.91s 93 | ``` 94 | 95 | * This Raspberry Pi has 3 cores. 96 | * Python 3.11 is significantly faster than 3.9. 97 | 98 | ## Sequential benchmarks 99 | 100 | ### Python 3.10 v. PyPy3 3.8 [7.3] 101 | 102 | 103 | ``` 104 | $ python3 --version 105 | Python 3.10.12 106 | $ time python3 primes.py 107 | 2 True 108 | 142702110479723 True 109 | 299593572317531 True 110 | 3333333333333301 True 111 | 3333333333333333 False 112 | 3333335652092209 False 113 | 4444444444444423 True 114 | 4444444444444444 False 115 | 4444444488888889 False 116 | 5555553133149889 False 117 | 5555555555555503 True 118 | 5555555555555555 False 119 | 6666666666666666 False 120 | 6666666666666719 True 121 | 6666667141414921 False 122 | 7777777536340681 False 123 | 7777777777777753 True 124 | 7777777777777777 False 125 | 9999999999999917 True 126 | 9999999999999999 False 127 | 128 | real 0m12.517s 129 | user 0m12.517s 130 | sys 0m0.000s 131 | 132 | $ time pypy3 primes.py > /dev/null 133 | 134 | real 0m0.631s 135 | user 0m0.621s 136 | sys 0m0.010s 137 | 138 | ``` 139 | 140 | ### Raspberri Pi 4 (8GB) 141 | 142 | ``` 143 | $ python3 --version 144 | Python 3.9.2 145 | $ time python3 primes.py > /dev/null 146 | 147 | real 2m0.255s 148 | user 0m0.204s 149 | sys 0m0.029s 150 | ``` 151 | 152 | 153 | ### Python v. Go 154 | 155 | ``` 156 | $ time python3.10 primes.py > /dev/null 157 | 158 | real 0m13.432s 159 | user 0m13.432s 160 | sys 0m0.000s 161 | 162 | $ time pypy3 primes.py > /dev/null 163 | 164 | real 0m0.643s 165 | user 0m0.643s 166 | sys 0m0.000s 167 | 168 | $ time go run primes.go > /dev/null 169 | 170 | real 0m0.767s 171 | user 0m0.728s 172 | sys 0m0.123s 173 | 174 | $ go build primes.go 175 | $ time ./primes > /dev/null 176 | 177 | real 0m0.600s 178 | user 0m0.601s 179 | sys 0m0.000s 180 | ``` -------------------------------------------------------------------------------- /concorrencia/primes/future_procs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import sys 5 | import time 6 | from concurrent import futures 7 | 8 | from primes import least_prime_factor, make_sample, LeastPrimeFactor 9 | 10 | # Magnitude of primes that take a few seconds to check 11 | # 12 | # machine magnitude 13 | # RPI4 2 ** 49 14 | # X250 2 ** 53 15 | # YOGA9 2 ** 57 16 | # M2MAX 2 ** 57 17 | # VIVO 2 ** 63 18 | 19 | MAGNITUDE = 2**57 20 | 21 | 22 | def check_lpf(n: int) -> tuple[LeastPrimeFactor, float]: 23 | t0 = time.perf_counter() 24 | lpf = LeastPrimeFactor(n, least_prime_factor(n)) 25 | elapsed = time.perf_counter() - t0 26 | return (lpf, elapsed) 27 | 28 | 29 | def main(): 30 | if len(sys.argv) < 2: # <1> 31 | qtd_procs = os.cpu_count() 32 | else: 33 | qtd_procs = int(sys.argv[1]) 34 | 35 | sample = make_sample(MAGNITUDE) 36 | t0 = time.perf_counter() 37 | processing_time = 0 38 | with futures.ProcessPoolExecutor(max_workers=qtd_procs) as executor: 39 | tasks = (executor.submit(check_lpf, n) for n in sample) 40 | for future in futures.as_completed(tasks): 41 | lpf, elapsed = future.result() 42 | separator = '=' if lpf.prime else ' ' 43 | print(f'{lpf.n:26_d} {separator} {lpf.factor:26_d} ({elapsed:9.5f}s)') 44 | processing_time += elapsed 45 | elapsed = time.perf_counter() - t0 46 | print(f'{len(sample)} checks in {elapsed:.1f}s') 47 | print(f'Total processing time: {processing_time:.1f}s using {qtd_procs} processes.') 48 | 49 | 50 | if __name__ == '__main__': 51 | main() 52 | -------------------------------------------------------------------------------- /concorrencia/primes/future_threads.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import sys 5 | import time 6 | from concurrent import futures 7 | 8 | from primes import least_prime_factor, make_sample, LeastPrimeFactor 9 | 10 | # Magnitude of primes that take a few seconds to check 11 | # 12 | # machine magnitude 13 | # RPI4 2 ** 49 14 | # X250 2 ** 53 15 | # YOGA9 2 ** 57 16 | # M2MAX 2 ** 57 17 | # VIVO 2 ** 63 18 | 19 | MAGNITUDE = 2**57 20 | 21 | 22 | def check_lpf(n: int) -> tuple[LeastPrimeFactor, float]: 23 | t0 = time.perf_counter() 24 | lpf = LeastPrimeFactor(n, least_prime_factor(n)) 25 | elapsed = time.perf_counter() - t0 26 | return (lpf, elapsed) 27 | 28 | 29 | def main(): 30 | if len(sys.argv) < 2: # <1> 31 | qtd_procs = os.cpu_count() 32 | else: 33 | qtd_procs = int(sys.argv[1]) 34 | 35 | sample = make_sample(MAGNITUDE) 36 | t0 = time.perf_counter() 37 | processing_time = 0 38 | 39 | with futures.ThreadPoolExecutor(max_workers=qtd_procs) as executor: 40 | tasks = (executor.submit(check_lpf, n) for n in sample) 41 | for future in futures.as_completed(tasks): 42 | lpf, elapsed = future.result() 43 | separator = '=' if lpf.prime else ' ' 44 | print(f'{lpf.n:26_d} {separator} {lpf.factor:26_d} ({elapsed:9.5f}s)') 45 | processing_time += elapsed 46 | elapsed = time.perf_counter() - t0 47 | print(f'{len(sample)} checks in {elapsed:.1f}s') 48 | print(f'Total processing time: {processing_time:.1f}s using {qtd_procs} threads.') 49 | 50 | 51 | if __name__ == '__main__': 52 | main() 53 | -------------------------------------------------------------------------------- /concorrencia/primes/integers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Interesting integers for experiments 5 | """ 6 | 7 | import sys 8 | 9 | 10 | MAX_UINT64 = 18_446_744_073_709_551_615 # from Go: ^uint64(0) 11 | assert MAX_UINT64 == 2**64 - 1 12 | 13 | print(f'{MAX_UINT64:26d} // max uint64') 14 | print() 15 | for i, j in enumerate(range(29, 65, 5), 1): 16 | n = 2**j 17 | # print(f'{i:2d}', end = ' ') 18 | print(f'{n:26d}, // 2 ** {j}') 19 | -------------------------------------------------------------------------------- /concorrencia/primes/map_primes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import sys 5 | import time 6 | from concurrent import futures 7 | 8 | from primes import least_prime_factor, make_sample, LeastPrimeFactor 9 | 10 | # Magnitude of primes that take a few seconds to check 11 | # 12 | # machine magnitude 13 | # RPI4 2 ** 49 14 | # X250 2 ** 53 15 | # YOGA9 2 ** 57 16 | # M2MAX 2 ** 57 17 | # VIVO 2 ** 63 18 | 19 | MAGNITUDE = 2**57 20 | 21 | 22 | def check_lpf(n: int) -> tuple[LeastPrimeFactor, float]: 23 | t0 = time.perf_counter() 24 | lpf = LeastPrimeFactor(n, least_prime_factor(n)) 25 | elapsed = time.perf_counter() - t0 26 | return (lpf, elapsed) 27 | 28 | 29 | def main(): 30 | if len(sys.argv) < 2: # <1> 31 | qtd_procs = os.cpu_count() 32 | else: 33 | qtd_procs = int(sys.argv[1]) 34 | 35 | sample = make_sample(MAGNITUDE) 36 | t0 = time.perf_counter() 37 | processing_time = 0 38 | with futures.ProcessPoolExecutor(max_workers=qtd_procs) as executor: 39 | for lpf, elapsed in executor.map(check_lpf, sample): 40 | separator = '=' if lpf.prime else ' ' 41 | print(f'{lpf.n:26_d} {separator} {lpf.factor:26_d} ({elapsed:9.5f}s)') 42 | processing_time += elapsed 43 | elapsed = time.perf_counter() - t0 44 | print(f'{len(sample)} checks in {elapsed:.1f}s') 45 | print(f'Total processing time: {processing_time:.1f}s using {qtd_procs} processes.') 46 | 47 | 48 | if __name__ == '__main__': 49 | main() 50 | -------------------------------------------------------------------------------- /concorrencia/primes/n_primes_proc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | n_primes_proc.py: shows that multiprocessing on a multicore machine 5 | can be faster than sequential code for CPU-intensive work. 6 | 7 | This is a DIY implementation of the worker pool pattern: the main 8 | program starts a number of worker processes which read integers 9 | from a queue, checks whether each integer is prime, and and posts 10 | results into another queue which is consumed by a function that 11 | displays the results. 12 | """ 13 | 14 | import sys 15 | from time import perf_counter 16 | from typing import NamedTuple 17 | from multiprocessing import Process, SimpleQueue, cpu_count # <1> 18 | from multiprocessing import queues # <2> 19 | 20 | from primes import least_prime_factor, make_sample 21 | 22 | """ 23 | Using `fork` to fix FileNotFoundError happening on MacOS 13.6 (Ventura) 24 | using Python 3.11 or 3.12.0 installed from images on python.org. 25 | The FileNotFoundError does not happen using PyPy 7.3.12 (~Python 3.10.12). 26 | 27 | Traceback (most recent call last): 28 | File "", line 1, in 29 | File "/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/spawn.py", line 122, in spawn_main 30 | exitcode = _main(fd, parent_sentinel) 31 | ^^^^^^^^^^^^^^^^^^^^^^^^^^ 32 | File "/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/spawn.py", line 132, in _main 33 | self = reduction.pickle.load(from_parent) 34 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 35 | File "/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/synchronize.py", line 115, in __setstate__ 36 | self._semlock = _multiprocessing.SemLock._rebuild(*state) 37 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 38 | FileNotFoundError: [Errno 2] No such file or directory 39 | 40 | Fix mentioned here: 41 | https://superfastpython.com/filenotfounderror-multiprocessing-python/ 42 | 43 | """ 44 | from multiprocessing import set_start_method 45 | 46 | set_start_method('fork') 47 | 48 | # Magnitude of primes that take a few seconds to check 49 | # 50 | # machine magnitude 51 | # RPI4 2 ** 49 52 | # X250 2 ** 53 53 | # YOGA9 2 ** 57 54 | # M2MAX 2 ** 57 55 | # VIVO 2 ** 63 56 | 57 | MAGNITUDE = 2**57 58 | 59 | 60 | class Experiment(NamedTuple): # <3> 61 | n: int 62 | lpf: int 63 | elapsed: float 64 | 65 | @property 66 | def prime(self): 67 | return self.n == self.lpf 68 | 69 | 70 | JobQueue = queues.SimpleQueue[int] # <4> 71 | ResultQueue = queues.SimpleQueue[Experiment] # <5> 72 | 73 | 74 | def check(n: int) -> Experiment: # <6> 75 | t0 = perf_counter() 76 | res = least_prime_factor(n) 77 | return Experiment(n, res, perf_counter() - t0) 78 | 79 | 80 | def worker(jobs: JobQueue, results: ResultQueue) -> None: # <7> 81 | while n := jobs.get(): # <8> 82 | results.put(check(n)) # <9> 83 | results.put(Experiment(0, False, 0.0)) # <10> 84 | 85 | 86 | def start_jobs(qtd_procs: int, results: ResultQueue) -> None: 87 | jobs: JobQueue = SimpleQueue() # <2> 88 | for n in make_sample(MAGNITUDE): 89 | jobs.put(n) # <12> 90 | for _ in range(qtd_procs): 91 | proc = Process(target=worker, args=(jobs, results)) # <13> 92 | proc.start() # <14> 93 | jobs.put(0) # <15> "poison pill" 94 | 95 | 96 | def report(qtd_procs: int, results: ResultQueue) -> int: # <6> 97 | checked = 0 98 | procs_done = 0 99 | while procs_done < qtd_procs: # <7> 100 | exp = results.get() # <8> 101 | if exp.n == 0: # <9> 102 | procs_done += 1 103 | else: 104 | checked += 1 # <10> 105 | label = '=' if exp.prime else ' ' 106 | print(f'{exp.n:26_d} {label} {exp.lpf:26_d} {exp.elapsed:9.6f}s') 107 | return checked 108 | 109 | 110 | def main() -> None: 111 | if len(sys.argv) < 2: # <1> 112 | qtd_procs = cpu_count() 113 | else: 114 | qtd_procs = int(sys.argv[1]) 115 | 116 | print(f'Using {qtd_procs} worker processes.') 117 | t0 = perf_counter() 118 | results: ResultQueue = SimpleQueue() 119 | start_jobs(qtd_procs, results) # <3> 120 | checked = report(qtd_procs, results) # <4> 121 | elapsed = perf_counter() - t0 122 | print(f'{checked} checks in {elapsed:.1f}s') # <5> 123 | 124 | if __name__ == '__main__': 125 | main() 126 | -------------------------------------------------------------------------------- /concorrencia/primes/prime-cookies.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramalho/python-eng/59a007d54595f11441ec7de07909576e1c158772/concorrencia/primes/prime-cookies.png -------------------------------------------------------------------------------- /concorrencia/primes/primes-demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "b7f93d8d-010e-4843-8b50-8fb6b94fb3b2", 6 | "metadata": {}, 7 | "source": [ 8 | "# Demonstrações da live #2" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": 1, 14 | "id": "c93bb55c-7a43-4710-882d-fddbe38c1caf", 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "import os, math\n", 19 | "from random import shuffle\n", 20 | "\n", 21 | "from primes import least_prime_factor, make_sample\n", 22 | "from table import Table\n", 23 | "\n", 24 | "# Magnitude of primes that take a few seconds to check\n", 25 | "#\n", 26 | "# machine magnitude\n", 27 | "# RPI4 2 ** 49\n", 28 | "# X250 2 ** 53\n", 29 | "# YOGA9 2 ** 57\n", 30 | "# M2MAX 2 ** 57\n", 31 | "# VIVO 2 ** 63\n", 32 | "\n", 33 | "MAGNITUDE = 2**57\n", 34 | "\n", 35 | "sample = sorted(make_sample(MAGNITUDE))" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 2, 41 | "id": "ae31456f-2b81-4a68-94c1-619d423d836f", 42 | "metadata": {}, 43 | "outputs": [ 44 | { 45 | "data": { 46 | "application/vnd.jupyter.widget-view+json": { 47 | "model_id": "5a5adfa4fab6418591d4adb26d583b66", 48 | "version_major": 2, 49 | "version_minor": 0 50 | }, 51 | "text/plain": [ 52 | "VBox(children=(Valid(value=False, description='18_014_398_509_481_951', layout=Layout(width='90%'), readout='⏳…" 53 | ] 54 | }, 55 | "metadata": {}, 56 | "output_type": "display_data" 57 | }, 58 | { 59 | "name": "stdout", 60 | "output_type": "stream", 61 | "text": [ 62 | "SAMPLE SIZE: 21 MAGNITUDE: 10**17 WORKERS: 1\n", 63 | "CPU times: user 33.7 s, sys: 40.8 ms, total: 33.7 s\n", 64 | "Wall time: 34.3 s\n" 65 | ] 66 | } 67 | ], 68 | "source": [ 69 | "%%time\n", 70 | "\n", 71 | "table = Table(sample)\n", 72 | "table.display()\n", 73 | "\n", 74 | "def update_table():\n", 75 | " for n in sample:\n", 76 | " lpf = least_prime_factor(n)\n", 77 | " table.update(n, lpf)\n", 78 | "\n", 79 | "update_table()\n", 80 | "workers = 1\n", 81 | "print(f'SAMPLE SIZE: {len(sample)} MAGNITUDE: 10**{round(math.log10(MAGNITUDE))} WORKERS: {workers}')" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": 3, 87 | "id": "596b71fc-a167-428e-bf33-7590b80e92d8", 88 | "metadata": {}, 89 | "outputs": [], 90 | "source": [ 91 | "from concurrent import futures\n", 92 | "\n", 93 | "from primes import lpf_pair " 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": 4, 99 | "id": "54a13f93-08fa-46b2-aac4-2a1a1f9576f2", 100 | "metadata": {}, 101 | "outputs": [ 102 | { 103 | "data": { 104 | "application/vnd.jupyter.widget-view+json": { 105 | "model_id": "e56a6a66842142c79d6e133942060b39", 106 | "version_major": 2, 107 | "version_minor": 0 108 | }, 109 | "text/plain": [ 110 | "VBox(children=(Valid(value=False, description='18_014_398_509_481_951', layout=Layout(width='90%'), readout='⏳…" 111 | ] 112 | }, 113 | "metadata": {}, 114 | "output_type": "display_data" 115 | }, 116 | { 117 | "name": "stdout", 118 | "output_type": "stream", 119 | "text": [ 120 | "SAMPLE SIZE: 21 MAGNITUDE: 10**17 WORKERS: 12\n", 121 | "CPU times: user 68 ms, sys: 45.7 ms, total: 114 ms\n", 122 | "Wall time: 6.48 s\n" 123 | ] 124 | } 125 | ], 126 | "source": [ 127 | "%%time\n", 128 | "\n", 129 | "table = Table(sample)\n", 130 | "table.display()\n", 131 | "\n", 132 | "with futures.ProcessPoolExecutor() as pool:\n", 133 | " tasks = [pool.submit(lpf_pair, n) for n in sample]\n", 134 | " for future in futures.as_completed(tasks):\n", 135 | " n, lpf = future.result()\n", 136 | " table.update(n, lpf)\n", 137 | "\n", 138 | "workers = pool._max_workers\n", 139 | "print(f'SAMPLE SIZE: {len(sample)} MAGNITUDE: 10**{round(math.log10(MAGNITUDE))} WORKERS: {workers}')" 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": null, 145 | "id": "7d2a2e9b-047d-4b9c-bb22-68bb45557004", 146 | "metadata": {}, 147 | "outputs": [], 148 | "source": [ 149 | "# desempenho pior com threads:\n", 150 | "#with futures.ThreadPoolExecutor() as pool: " 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": null, 156 | "id": "810f8e8a-629e-416f-ad07-7ef4fa9e23b4", 157 | "metadata": {}, 158 | "outputs": [], 159 | "source": [] 160 | } 161 | ], 162 | "metadata": { 163 | "kernelspec": { 164 | "display_name": "Python 3 (ipykernel)", 165 | "language": "python", 166 | "name": "python3" 167 | }, 168 | "language_info": { 169 | "codemirror_mode": { 170 | "name": "ipython", 171 | "version": 3 172 | }, 173 | "file_extension": ".py", 174 | "mimetype": "text/x-python", 175 | "name": "python", 176 | "nbconvert_exporter": "python", 177 | "pygments_lexer": "ipython3", 178 | "version": "3.12.2" 179 | } 180 | }, 181 | "nbformat": 4, 182 | "nbformat_minor": 5 183 | } 184 | -------------------------------------------------------------------------------- /concorrencia/primes/primeserver/go.mod: -------------------------------------------------------------------------------- 1 | module primeserver 2 | 3 | go 1.21.1 4 | -------------------------------------------------------------------------------- /concorrencia/primes/primeserver/server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "math" 7 | "math/big" 8 | "net/http" 9 | "strconv" 10 | ) 11 | 12 | type responseLPF struct { 13 | N uint64 `json:"n"` 14 | LPF uint64 `json:"lpf"` 15 | } 16 | 17 | func main() { 18 | http.HandleFunc("/lpf", getLPF) 19 | http.ListenAndServe(":8000", nil) 20 | } 21 | 22 | func getLPF(w http.ResponseWriter, r *http.Request) { 23 | q := r.URL.Query().Get("n") 24 | if len(q) == 0 { 25 | w.WriteHeader(http.StatusBadRequest) 26 | w.Write([]byte("missing query value n=")) 27 | return 28 | } 29 | 30 | n, err := strconv.ParseUint(q, 10, 64) 31 | if err != nil { 32 | w.WriteHeader(http.StatusBadRequest) 33 | w.Write([]byte(fmt.Sprintf("invalid query string n=%v", q))) 34 | return 35 | } 36 | jsonBytes, err := json.Marshal(responseLPF{n, LPF(n)}) 37 | if err != nil { 38 | w.WriteHeader(http.StatusInternalServerError) 39 | w.Write([]byte("internal server error")) 40 | return 41 | } 42 | w.WriteHeader(http.StatusOK) 43 | w.Write(jsonBytes) 44 | } 45 | 46 | func LPF(n uint64) uint64 { 47 | switch { 48 | case n == 1: 49 | return 1 50 | case n%2 == 0: 51 | return 2 52 | case n%3 == 0: 53 | return 3 54 | } 55 | 56 | // n.ProbablyPrime(0) uses the Baillie-PSW primality test 57 | // https://en.wikipedia.org/wiki/Baillie%E2%80%93PSW_primality_test 58 | if n < math.MaxInt64 && big.NewInt(int64(n)).ProbablyPrime(0) { 59 | return n 60 | } 61 | limit := uint64(math.Sqrt(float64(n))) 62 | for i := uint64(5); i <= limit; i += 6 { 63 | if n%i == 0 { 64 | return i 65 | } 66 | j := i + 2 67 | if n%j == 0 { 68 | return j 69 | } 70 | } 71 | return n 72 | } 73 | -------------------------------------------------------------------------------- /concorrencia/primes/process-workflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramalho/python-eng/59a007d54595f11441ec7de07909576e1c158772/concorrencia/primes/process-workflow.png -------------------------------------------------------------------------------- /concorrencia/primes/table.py: -------------------------------------------------------------------------------- 1 | """ 2 | `Table` manages a list of `Valid` widgets, each representing a number in the sample. 3 | """ 4 | 5 | from IPython.display import display 6 | from ipywidgets.widgets import Valid, VBox, Layout 7 | 8 | class Table: 9 | def __init__(self, sample): 10 | self.layout = Layout(width='90%') 11 | self.style = {'description_width' : '200px'} 12 | self.cells = [self.make_cell(n, 0) for n in sample] 13 | self.cell_map = {n:i for i, n in enumerate(sample)} 14 | self.box = VBox(self.cells) 15 | 16 | def make_cell(self, n, lpf, elapsed=None): 17 | if elapsed is None: 18 | result = f'{lpf:_}' if lpf else '\N{Hourglass with Flowing Sand}' 19 | else: 20 | result = f'{lpf:_} ({elapsed:.3f}s)' if lpf != n else f'({elapsed:.3f}s)' 21 | return Valid( 22 | value = lpf == n, 23 | description = f'{n:_}', 24 | readout = result, 25 | layout = self.layout, 26 | style = self.style, 27 | ) 28 | 29 | def display(self): 30 | display(self.box) 31 | 32 | def update(self, n, lpf, elapsed=None): 33 | self.cells[self.cell_map[n]] = self.make_cell(n, lpf, elapsed) 34 | self.box.children = self.cells 35 | -------------------------------------------------------------------------------- /concorrencia/primes/time_primes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import time 5 | from operator import attrgetter 6 | from primes import EXPERIMENTS, is_prime 7 | 8 | __doc__ = f""" 9 | {sys.argv[0]} finds a prime number that takes "about" TIME seconds to check. 10 | 11 | USAGE: 12 | {sys.argv[0]} TIME 13 | """.strip() 14 | 15 | primes = [exp.n for exp in EXPERIMENTS if exp.prime] 16 | primes.sort() 17 | 18 | 19 | def time_cost(n): 20 | t0 = time.perf_counter() 21 | is_prime(n) 22 | return time.perf_counter() - t0 23 | 24 | 25 | def main(): 26 | try: 27 | target_time = float(sys.argv[1]) 28 | except (IndexError, ValueError): 29 | print(__doc__) 30 | sys.exit(-1) 31 | 32 | start = 0 33 | end = len(primes) 34 | delta = sys.maxsize 35 | 36 | while end - start > 1: 37 | middle = (start + end) // 2 38 | pick = primes[middle] 39 | print(f'{start:2}:{end:<2} [{middle:2}] {pick:26_}', end=' ', flush=True) 40 | t = time_cost(pick) 41 | print(f'({t:.3f}s)') 42 | if t <= target_time: 43 | start = middle 44 | else: 45 | end = middle 46 | 47 | marker = '^' * len(f'{pick:_}') 48 | print(f'Closest prime {marker:>26}') 49 | 50 | 51 | if __name__ == '__main__': 52 | main() 53 | -------------------------------------------------------------------------------- /concorrencia/primes/widget-lab.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "c93bb55c-7a43-4710-882d-fddbe38c1caf", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "from random import shuffle\n", 11 | "\n", 12 | "from IPython.display import display\n", 13 | "from ipywidgets.widgets import Valid, VBox, Layout\n", 14 | "\n", 15 | "from primes import least_prime_factor, make_sample\n", 16 | "\n", 17 | "# Magnitude of primes that take a few seconds to check\n", 18 | "#\n", 19 | "# machine magnitude\n", 20 | "# RPI4 2 ** 49\n", 21 | "# X250 2 ** 53\n", 22 | "# YOGA9 2 ** 57\n", 23 | "# M2MAX 2 ** 57\n", 24 | "# VIVO 2 ** 63\n", 25 | "\n", 26 | "MAGNITUDE = 2**57\n", 27 | "\n", 28 | "sample = make_sample(MAGNITUDE)\n", 29 | "shuffle(sample)\n", 30 | "\n", 31 | "def make_cell(n, lpf):\n", 32 | " return Valid(\n", 33 | " value = lpf == n,\n", 34 | " description = f'{n:_}',\n", 35 | " readout = f'{lpf:_}' if lpf else '\\N{Hourglass with Flowing Sand}',\n", 36 | " layout = Layout(width='90%'),\n", 37 | " style = {'description_width' : '200px'},\n", 38 | " )" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": 2, 44 | "id": "ae31456f-2b81-4a68-94c1-619d423d836f", 45 | "metadata": {}, 46 | "outputs": [ 47 | { 48 | "data": { 49 | "application/vnd.jupyter.widget-view+json": { 50 | "model_id": "43ca7a5719864af78b07a0aa38cbc3a9", 51 | "version_major": 2, 52 | "version_minor": 0 53 | }, 54 | "text/plain": [ 55 | "VBox(children=(Valid(value=False, description='18_014_399_314_786_597', layout=Layout(width='90%'), readout='⏳…" 56 | ] 57 | }, 58 | "metadata": {}, 59 | "output_type": "display_data" 60 | }, 61 | { 62 | "name": "stdout", 63 | "output_type": "stream", 64 | "text": [ 65 | "CPU times: user 33.1 s, sys: 19 ms, total: 33.2 s\n", 66 | "Wall time: 33.3 s\n" 67 | ] 68 | } 69 | ], 70 | "source": [ 71 | "%%time\n", 72 | "cells = [make_cell(n, 0) for n in sample]\n", 73 | "\n", 74 | "table = VBox(cells) \n", 75 | "display(table)\n", 76 | "\n", 77 | "def update_table():\n", 78 | " for i, n in enumerate(sample):\n", 79 | " lpf = least_prime_factor(n)\n", 80 | " cells[i] = make_cell(n, lpf)\n", 81 | " table.children = cells\n", 82 | "\n", 83 | "update_table()" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 3, 89 | "id": "596b71fc-a167-428e-bf33-7590b80e92d8", 90 | "metadata": {}, 91 | "outputs": [], 92 | "source": [ 93 | "from concurrent import futures\n", 94 | "\n", 95 | "from primes import lpf_pair\n", 96 | "from table import Table\n", 97 | " " 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": 4, 103 | "id": "54a13f93-08fa-46b2-aac4-2a1a1f9576f2", 104 | "metadata": {}, 105 | "outputs": [ 106 | { 107 | "data": { 108 | "application/vnd.jupyter.widget-view+json": { 109 | "model_id": "95e480089e9949a3b85785b6438ed671", 110 | "version_major": 2, 111 | "version_minor": 0 112 | }, 113 | "text/plain": [ 114 | "VBox(children=(Valid(value=False, description='18_014_399_314_786_597', layout=Layout(width='90%'), readout='⏳…" 115 | ] 116 | }, 117 | "metadata": {}, 118 | "output_type": "display_data" 119 | }, 120 | { 121 | "name": "stdout", 122 | "output_type": "stream", 123 | "text": [ 124 | "CPU times: user 48 ms, sys: 35.1 ms, total: 83.1 ms\n", 125 | "Wall time: 6.12 s\n" 126 | ] 127 | } 128 | ], 129 | "source": [ 130 | "%%time\n", 131 | "\n", 132 | "table = Table(sample)\n", 133 | "table.display()\n", 134 | "\n", 135 | "with futures.ProcessPoolExecutor() as pool:\n", 136 | " tasks = [pool.submit(lpf_pair, n) for n in sample]\n", 137 | " for future in futures.as_completed(tasks):\n", 138 | " n, lpf = future.result()\n", 139 | " table.update(n, lpf)" 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": null, 145 | "id": "7d2a2e9b-047d-4b9c-bb22-68bb45557004", 146 | "metadata": {}, 147 | "outputs": [], 148 | "source": [] 149 | } 150 | ], 151 | "metadata": { 152 | "kernelspec": { 153 | "display_name": "Python 3 (ipykernel)", 154 | "language": "python", 155 | "name": "python3" 156 | }, 157 | "language_info": { 158 | "codemirror_mode": { 159 | "name": "ipython", 160 | "version": 3 161 | }, 162 | "file_extension": ".py", 163 | "mimetype": "text/x-python", 164 | "name": "python", 165 | "nbconvert_exporter": "python", 166 | "pygments_lexer": "ipython3", 167 | "version": "3.12.2" 168 | } 169 | }, 170 | "nbformat": 4, 171 | "nbformat_minor": 5 172 | } 173 | -------------------------------------------------------------------------------- /concorrencia/processos-e-threads.odp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramalho/python-eng/59a007d54595f11441ec7de07909576e1c158772/concorrencia/processos-e-threads.odp -------------------------------------------------------------------------------- /concorrencia/processos-e-threads.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramalho/python-eng/59a007d54595f11441ec7de07909576e1c158772/concorrencia/processos-e-threads.pdf -------------------------------------------------------------------------------- /concorrencia/spin_proc.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | from multiprocessing import Process, Event 3 | 4 | from primes import is_prime 5 | 6 | def girar(msg: str, pronto: Event) -> None: 7 | for char in itertools.cycle(r'\|/-'): 8 | status = f'\r{char} {msg}' 9 | print(status, end='', flush=True) 10 | if pronto.wait(.05): 11 | break 12 | blanks = ' ' * len(status) 13 | print(f'\r{blanks}\r', end='') 14 | 15 | 16 | def supervisor(): 17 | pronto = Event() 18 | girador = Process(target=girar, args=['Computando...', pronto]) 19 | girador.start() 20 | n = 7_777_777_777_777_753 21 | primo = is_prime(n) 22 | pronto.set() 23 | girador.join() 24 | e_nao_e = 'é' if primo else 'não é' 25 | print(n, e_nao_e, 'primo' ) 26 | 27 | 28 | if __name__ == '__main__': 29 | supervisor() 30 | -------------------------------------------------------------------------------- /concorrencia/wikipics/README.md: -------------------------------------------------------------------------------- 1 | # Wikipics: downloading pictures from Wikipedia 2 | 3 | The main examples are named `download_*.py`. 4 | 5 | Most of the other files have to do with getting sample URLs 6 | to download Wikipedia images of an approximate size, to 7 | make demonstrations that are not too quick nor too slow. 8 | 9 | 10 | ## Starting files 11 | 12 | * `spinner.py`: example of character-based animation on the terminal 13 | * `wikipics.py`: library to get URLs of sample images; function `get_sample_url(size)` returns the URL of a file of approximately `size` bytes. -------------------------------------------------------------------------------- /concorrencia/wikipics/coro_1_download.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import asyncio 4 | import itertools 5 | import time 6 | 7 | from pathlib import Path 8 | 9 | import httpx 10 | 11 | import wikipics 12 | 13 | SAVE_DIR = Path('img') 14 | 15 | # https://en.wikipedia.org/wiki/Braille_Patterns 16 | SPRITE = [ 17 | '\N{BRAILLE PATTERN DOTS-14}', 18 | '\N{BRAILLE PATTERN DOTS-25}', 19 | '\N{BRAILLE PATTERN DOTS-36}', 20 | '\N{BRAILLE PATTERN DOTS-78}', 21 | ] 22 | 23 | 24 | async def spin(msg: str) -> None: 25 | for char in itertools.cycle(SPRITE): 26 | status = f'\r{char} {msg}' 27 | print(status, end='', flush=True) 28 | try: 29 | await asyncio.sleep(0.1) 30 | except asyncio.CancelledError: 31 | break 32 | blanks = ' ' * len(status) 33 | print(f'\r{blanks}\r', end='') 34 | 35 | 36 | async def fetch(url) -> bytes: 37 | async with httpx.AsyncClient() as client: 38 | resp = await client.get(url) 39 | resp.raise_for_status() 40 | return resp.content 41 | 42 | 43 | async def download(url) -> tuple[int, str]: 44 | octets = await fetch(url) 45 | name = Path(url).name 46 | with open(SAVE_DIR / name, 'wb') as fp: 47 | fp.write(octets) 48 | return len(octets), SAVE_DIR / name 49 | 50 | 51 | async def supervisor() -> str: 52 | spinner = asyncio.create_task(spin('downloading')) 53 | print(f'spinner object: {spinner}') 54 | url = wikipics.get_sample_url(40_000_000) 55 | t0 = time.perf_counter() 56 | size, name = await download(url) 57 | dt = time.perf_counter() - t0 58 | spinner.cancel() 59 | return f'{size:_d} bytes in {dt:0.1f}s\n{name}' 60 | 61 | 62 | def main(): 63 | msg = asyncio.run(supervisor()) 64 | print(msg) 65 | 66 | 67 | if __name__ == '__main__': 68 | main() 69 | -------------------------------------------------------------------------------- /concorrencia/wikipics/de-gera-a-coro.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "349495a0-c287-402b-8cce-fec30e817b78", 6 | "metadata": {}, 7 | "source": [ 8 | "# De geradores a corrotinas" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": 1, 14 | "id": "371af48e-1acc-409b-a3a2-4994714751d5", 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "def gen123():\n", 19 | " yield 1\n", 20 | " yield 2\n", 21 | " yield 3" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 2, 27 | "id": "20741abf-0f0b-423d-a242-dd6cd3b78168", 28 | "metadata": {}, 29 | "outputs": [ 30 | { 31 | "data": { 32 | "text/plain": [ 33 | "[1, 2, 3]" 34 | ] 35 | }, 36 | "execution_count": 2, 37 | "metadata": {}, 38 | "output_type": "execute_result" 39 | } 40 | ], 41 | "source": [ 42 | "list(gen123())" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": null, 48 | "id": "d3850b88-b5d8-40e4-be45-749c71cf10f3", 49 | "metadata": {}, 50 | "outputs": [], 51 | "source": [] 52 | } 53 | ], 54 | "metadata": { 55 | "kernelspec": { 56 | "display_name": "Python 3 (ipykernel)", 57 | "language": "python", 58 | "name": "python3" 59 | }, 60 | "language_info": { 61 | "codemirror_mode": { 62 | "name": "ipython", 63 | "version": 3 64 | }, 65 | "file_extension": ".py", 66 | "mimetype": "text/x-python", 67 | "name": "python", 68 | "nbconvert_exporter": "python", 69 | "pygments_lexer": "ipython3", 70 | "version": "3.12.2" 71 | } 72 | }, 73 | "nbformat": 4, 74 | "nbformat_minor": 5 75 | } 76 | -------------------------------------------------------------------------------- /concorrencia/wikipics/future_thread_downloads.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Demo: download images with concurrent.futures 5 | """ 6 | 7 | from concurrent.futures import ThreadPoolExecutor, as_completed 8 | from pathlib import Path 9 | import time 10 | 11 | import httpx 12 | 13 | import wikipics 14 | 15 | WANTED_SIZE = 10 * 2 ** 20 # 40 MB 16 | SAMPLE_LEN = 20 17 | SAVE_DIR = Path('img') 18 | 19 | def save(filename: str, data: bytes): 20 | if len(filename) > 255: # common OS limit 21 | filename = filename[:255] 22 | with open(filename, 'wb') as fp: 23 | fp.write(data) 24 | 25 | def download(url) -> tuple[wikipics.ImageDatum, float]: 26 | t0 = time.perf_counter() 27 | resp = httpx.get(url) 28 | resp.raise_for_status() 29 | name = Path(url).name 30 | save(str(SAVE_DIR / name), resp.content) 31 | elapsed = time.perf_counter() - t0 32 | img = wikipics.ImageDatum(len(resp.content), str(SAVE_DIR / name)) 33 | return (elapsed, img) 34 | 35 | 36 | def crop_path(path: str, max_len:int=65) -> str: 37 | if len(path) > max_len: 38 | return path[:max_len - 1] + '\N{HORIZONTAL ELLIPSIS}' 39 | return path 40 | 41 | 42 | def main(): 43 | sample = wikipics.get_sample_urls(WANTED_SIZE, SAMPLE_LEN) 44 | t0 = time.perf_counter() 45 | processing_time = 0 46 | qtd_threads = 0 47 | success_count = 0 48 | 49 | with ThreadPoolExecutor() as executor: 50 | # Submit the download tasks to the executor 51 | futures = [executor.submit(download, url) for url in sample] 52 | qtd_threads = executor._max_workers 53 | print(f'Downloading {len(futures)} images with up to {qtd_threads} threads') 54 | 55 | # Wait for the tasks to complete 56 | for future in as_completed(futures): # , timeout=10 57 | # Get the result of the task 58 | try: 59 | elapsed, img = future.result() 60 | except httpx.HTTPStatusError as exc: 61 | print(f'HTTP error {exc}') 62 | continue 63 | success_count += 1 64 | path = crop_path(img.path) 65 | print(f'[{elapsed:3.1f}s] {img.size:10_d} {path}') 66 | processing_time += elapsed 67 | elapsed = time.perf_counter() - t0 68 | print(f'{success_count} downloads in {elapsed:.1f}s') 69 | print(f'Total processing time: {processing_time:.1f}s using {qtd_threads} threads.') 70 | 71 | 72 | 73 | if __name__ == '__main__': 74 | main() -------------------------------------------------------------------------------- /concorrencia/wikipics/gallery.py: -------------------------------------------------------------------------------- 1 | from IPython.display import display 2 | from ipywidgets.widgets import Image, Layout, HBox 3 | 4 | with open('no-image.png', 'rb') as fp: 5 | NO_IMAGE = fp.read() 6 | 7 | class Gallery: 8 | def __init__(self, num_images): 9 | scale = f'{100//num_images}%' 10 | self.img_layout = Layout(width=scale, height=scale) 11 | self.img_widgets = [Image(value=NO_IMAGE, layout=self.img_layout)] * num_images 12 | self.box = HBox(self.img_widgets) 13 | self.size = 0 14 | 15 | def display(self): 16 | display(self.box) 17 | 18 | def update(self, index, pixels, name=''): 19 | self.size += len(pixels) 20 | self.img_widgets[index] = Image(value=pixels, alt=f'({index})', layout=self.img_layout) 21 | self.box.children = self.img_widgets 22 | print(f'({index+1:2}){len(pixels):12_} bytes | {name}', flush=True) 23 | -------------------------------------------------------------------------------- /concorrencia/wikipics/img/.gitignore: -------------------------------------------------------------------------------- 1 | * -------------------------------------------------------------------------------- /concorrencia/wikipics/img/README.md: -------------------------------------------------------------------------------- 1 | Directory where downloaded images are saved. -------------------------------------------------------------------------------- /concorrencia/wikipics/no-image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramalho/python-eng/59a007d54595f11441ec7de07909576e1c158772/concorrencia/wikipics/no-image.png -------------------------------------------------------------------------------- /concorrencia/wikipics/process_1_download.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import itertools 4 | import time 5 | 6 | from pathlib import Path 7 | from multiprocessing import Event, Process 8 | 9 | import httpx 10 | 11 | import wikipics 12 | 13 | SAVE_DIR = Path('img') 14 | 15 | # https://en.wikipedia.org/wiki/Braille_Patterns 16 | SPRITE = [ 17 | '\N{BRAILLE PATTERN DOTS-14}', 18 | '\N{BRAILLE PATTERN DOTS-25}', 19 | '\N{BRAILLE PATTERN DOTS-36}', 20 | '\N{BRAILLE PATTERN DOTS-78}', 21 | ] 22 | 23 | 24 | def spin(msg: str, completed: Event) -> None: 25 | for char in itertools.cycle(SPRITE): 26 | status = f'\r{char} {msg}' 27 | print(status, end='', flush=True) 28 | if completed.wait(0.1): 29 | break 30 | blanks = ' ' * len(status) 31 | print(f'\r{blanks}\r', end='') 32 | 33 | 34 | def download(url) -> tuple[int, str]: 35 | resp = httpx.get(url) 36 | resp.raise_for_status() 37 | name = Path(url).name 38 | with open(SAVE_DIR / name, 'wb') as fp: 39 | fp.write(resp.content) 40 | return len(resp.content), SAVE_DIR / name 41 | 42 | 43 | def main(): 44 | completed = Event() 45 | spinner = Process(target=spin, args=('downloading', completed)) 46 | print(f'spinner object: {spinner}') 47 | spinner.start() 48 | url = wikipics.get_sample_url(40_000_000) 49 | t0 = time.perf_counter() 50 | size, name = download(url) 51 | dt = time.perf_counter() - t0 52 | completed.set() 53 | spinner.join() 54 | print(f'{size:_d} bytes in {dt:0.1f}s\n{name}') 55 | 56 | 57 | if __name__ == '__main__': 58 | main() 59 | -------------------------------------------------------------------------------- /concorrencia/wikipics/spinner.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import itertools 4 | import time 5 | from typing import NoReturn 6 | 7 | SPRITE = r'\|/-' 8 | 9 | 10 | def spin(msg: str) -> NoReturn: 11 | try: 12 | for char in itertools.cycle(SPRITE): 13 | status = f'\r{char} {msg}' 14 | print(status, end='', flush=True) 15 | time.sleep(0.1) 16 | except KeyboardInterrupt: 17 | blanks = ' ' * len(status) 18 | print(f'\r{blanks}\r', end='') 19 | 20 | 21 | if __name__ == '__main__': 22 | spin('thinking forever...') 23 | -------------------------------------------------------------------------------- /concorrencia/wikipics/thread_1_download.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import itertools 4 | import time 5 | 6 | from pathlib import Path 7 | from threading import Event, Thread 8 | 9 | import httpx 10 | 11 | import wikipics 12 | 13 | SAVE_DIR = Path('img') 14 | 15 | # https://en.wikipedia.org/wiki/Braille_Patterns 16 | SPRITE = [ 17 | '\N{BRAILLE PATTERN DOTS-14}', 18 | '\N{BRAILLE PATTERN DOTS-25}', 19 | '\N{BRAILLE PATTERN DOTS-36}', 20 | '\N{BRAILLE PATTERN DOTS-78}', 21 | ] 22 | 23 | 24 | def spin(msg: str, completed: Event) -> None: 25 | for char in itertools.cycle(SPRITE): 26 | status = f'\r{char} {msg}' 27 | print(status, end='', flush=True) 28 | if completed.wait(0.1): 29 | break 30 | blanks = ' ' * len(status) 31 | print(f'\r{blanks}\r', end='') 32 | 33 | 34 | def download(url) -> tuple[int, str]: 35 | resp = httpx.get(url) 36 | resp.raise_for_status() 37 | name = Path(url).name 38 | with open(SAVE_DIR / name, 'wb') as fp: 39 | fp.write(resp.content) 40 | return len(resp.content), SAVE_DIR / name 41 | 42 | 43 | def main(): 44 | completed = Event() 45 | spinner = Thread(target=spin, args=('downloading', completed)) 46 | print(f'spinner object: {spinner}') 47 | spinner.start() 48 | url = wikipics.get_sample_url(40_000_000) 49 | t0 = time.perf_counter() 50 | size, name = download(url) 51 | dt = time.perf_counter() - t0 52 | completed.set() 53 | spinner.join() 54 | print(f'{size:_d} bytes in {dt:0.1f}s\n{name}') 55 | 56 | 57 | if __name__ == '__main__': 58 | main() 59 | -------------------------------------------------------------------------------- /concorrencia/wikipics/wiki_probe_time.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Measure download times to find image size that 4 | # takes approximately TARGET seconds to download. 5 | 6 | import math 7 | import time 8 | 9 | import httpx 10 | 11 | import wikipics 12 | 13 | 14 | TARGET = 3.5 # seconds of download time 15 | TOLERANCE = 0.2 # relative tolerance 16 | 17 | MEGABYTES = 1024 * 1024 18 | 19 | 20 | def fetch(url) -> bytes: 21 | resp = httpx.get(url, timeout=TARGET * 3) 22 | resp.raise_for_status() 23 | return resp.content 24 | 25 | 26 | def new_guess(lower: int, upper: int) -> int: 27 | offset = (upper - lower) // 2 28 | return lower + offset 29 | 30 | 31 | def probe(target: float, img_data: tuple[int, str]) -> int: 32 | lower = 0 33 | upper = len(img_data) 34 | while True: 35 | guess = new_guess(lower, upper) 36 | size, path = img_data[guess] 37 | print( 38 | f'[{guess}] {size/MEGABYTES:_.1f} MB: {path} ', flush=True, end='' 39 | ) 40 | t0 = time.perf_counter() 41 | content = fetch(wikipics.BASE_URL + path) 42 | dt = time.perf_counter() - t0 43 | print(f'{dt:0.2f}s') 44 | assert len(content) == size 45 | if math.isclose(target, dt, rel_tol=TOLERANCE): 46 | return (guess, dt) 47 | if dt < target: 48 | lower = guess 49 | else: 50 | upper = guess 51 | 52 | 53 | def main(): 54 | img_data = wikipics.load_img_data() 55 | guess, dt = probe(TARGET, img_data) 56 | size, _ = img_data[guess] 57 | print( 58 | f'{dt:0.2f}s for image sized {size:_d} B at index [{guess}] of {len(img_data)}.' 59 | ) 60 | 61 | 62 | if __name__ == '__main__': 63 | main() 64 | -------------------------------------------------------------------------------- /concorrencia/wikipics/wikipics-demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "8a9dc8cc-f490-4d13-9a4f-e8ae0ec9125f", 6 | "metadata": {}, 7 | "source": [ 8 | "# Wikipics demo" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": null, 14 | "id": "2039cf8d-2612-40d9-ac4d-13f281e273ec", 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "from pathlib import Path\n", 19 | "from typing import NamedTuple\n", 20 | "\n", 21 | "from IPython.display import display\n", 22 | "from ipywidgets.widgets import Image, Layout\n", 23 | "import httpx\n", 24 | "\n", 25 | "from wikipics import get_sample_url, get_sample_urls\n", 26 | "\n", 27 | "class ImageRecord(NamedTuple):\n", 28 | " pixels: bytes\n", 29 | " name: str\n", 30 | " size: int" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": null, 36 | "id": "edf23542-1f6f-493a-8bab-a115c1d6d2c5", 37 | "metadata": {}, 38 | "outputs": [], 39 | "source": [ 40 | "def fetch(url) -> ImageRecord:\n", 41 | " resp = httpx.get(url)\n", 42 | " resp.raise_for_status()\n", 43 | " name = Path(url).name\n", 44 | " return ImageRecord(resp.content, name, len(resp.content))" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "id": "3b054372-b50d-484a-a95f-2d02129021be", 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "url = get_sample_url(1_000_000)\n", 55 | "img_rec = fetch(url)\n", 56 | "print(f'{img_rec.size:12_} bytes | {img_rec.name}')\n", 57 | "Image(value=img_rec.pixels)" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": null, 63 | "id": "3f935c85-64fe-45b2-8af9-b23cdb4d2158", 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "%%time\n", 68 | "img_widgets = []\n", 69 | "qty = 10\n", 70 | "urls = get_sample_urls(2_000_000, qty)\n", 71 | "total_bytes = 0\n", 72 | "\n", 73 | "for url in urls:\n", 74 | " img_rec = fetch(url)\n", 75 | " total_bytes += img_rec.size\n", 76 | " display(Image(value=img_rec.pixels, layout=Layout(width='20%')))\n", 77 | " print(f'{img_rec.size:12_} bytes | {img_rec.name}')\n", 78 | "\n", 79 | "print(f'TOTAL BYTES: {total_bytes:_}')" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": null, 85 | "id": "6799b3f5-e23d-40eb-a657-4668bd1316c5", 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [ 89 | "from gallery import Gallery" 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "execution_count": null, 95 | "id": "7c40632c-0239-48b0-8c01-eff3b0f6a4d1", 96 | "metadata": {}, 97 | "outputs": [], 98 | "source": [ 99 | "%%time\n", 100 | "num_images = 10\n", 101 | "gallery = Gallery(num_images)\n", 102 | "gallery.display()\n", 103 | "\n", 104 | "urls = get_sample_urls(2_000_000, num_images)\n", 105 | "\n", 106 | "for i, url in enumerate(urls):\n", 107 | " img_rec = fetch(url)\n", 108 | " total_bytes += img_rec.size\n", 109 | " gallery.update(i, img_rec.pixels, url)\n", 110 | "\n", 111 | "print(f'TOTAL BYTES: {gallery.size:_}')" 112 | ] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "execution_count": null, 117 | "id": "e7b1c72e-75eb-47ca-bbe1-51a39c16eada", 118 | "metadata": {}, 119 | "outputs": [], 120 | "source": [ 121 | "%%time\n", 122 | "from concurrent import futures\n", 123 | "\n", 124 | "num_images = 10\n", 125 | "gallery = Gallery(num_images)\n", 126 | "gallery.display()\n", 127 | "\n", 128 | "urls = get_sample_urls(2_000_000, num_images)\n", 129 | "with futures.ThreadPoolExecutor() as pool:\n", 130 | " img_records = pool.map(fetch, urls)\n", 131 | " for i, img_rec in enumerate(img_records):\n", 132 | " gallery.update(i, img_rec.pixels, img_rec.name)\n", 133 | "\n", 134 | "print(f'TOTAL BYTES: {gallery.size:_}')" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": null, 140 | "id": "539a5b55-929b-4c69-90c5-3847f1cb7efb", 141 | "metadata": {}, 142 | "outputs": [], 143 | "source": [ 144 | "%%time\n", 145 | "from concurrent import futures\n", 146 | "\n", 147 | "num_images = 10\n", 148 | "gallery = Gallery(num_images)\n", 149 | "gallery.display()\n", 150 | "urls = get_sample_urls(2_000_000, num_images)\n", 151 | "\n", 152 | "with futures.ThreadPoolExecutor() as pool:\n", 153 | " tasks = [pool.submit(fetch, url) for url in urls]\n", 154 | " for i, future in enumerate(futures.as_completed(tasks)):\n", 155 | " img_rec = future.result()\n", 156 | " gallery.update(i, img_rec.pixels, img_rec.name)\n", 157 | " \n", 158 | "print(f'TOTAL BYTES: {gallery.size:_}')" 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": null, 164 | "id": "af3d0bd8-adfa-4235-aee4-030d859ef4a6", 165 | "metadata": {}, 166 | "outputs": [], 167 | "source": [] 168 | } 169 | ], 170 | "metadata": { 171 | "kernelspec": { 172 | "display_name": "Python 3 (ipykernel)", 173 | "language": "python", 174 | "name": "python3" 175 | }, 176 | "language_info": { 177 | "codemirror_mode": { 178 | "name": "ipython", 179 | "version": 3 180 | }, 181 | "file_extension": ".py", 182 | "mimetype": "text/x-python", 183 | "name": "python", 184 | "nbconvert_exporter": "python", 185 | "pygments_lexer": "ipython3", 186 | "version": "3.12.2" 187 | } 188 | }, 189 | "nbformat": 4, 190 | "nbformat_minor": 5 191 | } 192 | -------------------------------------------------------------------------------- /concorrencia/wikipics/wikipics.py: -------------------------------------------------------------------------------- 1 | # Functions to get URLs for a sample of Wikipedia 2 | # images of an approximate byte size 3 | 4 | import functools 5 | import random 6 | import bisect 7 | from typing import TypeAlias, NamedTuple 8 | 9 | URL: TypeAlias = str 10 | 11 | IMG_DATA = 'jpeg-by-size.txt' 12 | BASE_URL = 'https://upload.wikimedia.org/wikipedia/commons/' 13 | 14 | WANTED_SIZE = 45_000_000 # bytes 15 | 16 | class ImageDatum(NamedTuple): 17 | size: int 18 | path: str 19 | 20 | 21 | @functools.cache 22 | def load_img_data() -> list[ImageDatum]: 23 | img_data = [] 24 | with open(IMG_DATA) as fp: 25 | for line in fp: 26 | size_str, path = line.strip().split('\t') 27 | size = int(size_str) 28 | # print(f'{size:10}\t{path}') 29 | img_data.append(ImageDatum(size, path)) 30 | return img_data 31 | 32 | 33 | def get_sample_urls(wanted_size, quantity) -> list[URL]: 34 | img_data = load_img_data() 35 | wanted = ImageDatum(wanted_size, '') 36 | anchor = bisect.bisect(img_data, wanted) 37 | # get slice 4 times bigger than quantity 38 | start, end = (anchor - quantity * 2), (anchor + quantity * 2) 39 | big_slice = img_data[start:end] 40 | urls = [BASE_URL + path for _, path in big_slice] 41 | random.shuffle(urls) 42 | return urls[:quantity] 43 | 44 | 45 | def get_sample_url(wanted_size): 46 | return get_sample_urls(wanted_size, 20)[0] 47 | 48 | 49 | def main(): 50 | img_data = load_img_data() 51 | sample = get_sample_urls(WANTED_SIZE, 20, img_data) 52 | print(sample) 53 | 54 | 55 | if __name__ == '__main__': 56 | main() 57 | -------------------------------------------------------------------------------- /data-model/README.rst: -------------------------------------------------------------------------------- 1 | Sample code for Chapter 1 - "The Python Data Model" 2 | 3 | From the book "Fluent Python" by Luciano Ramalho (O'Reilly, 2015) 4 | http://shop.oreilly.com/product/0636920032519.do 5 | -------------------------------------------------------------------------------- /data-model/frenchdeck.doctest: -------------------------------------------------------------------------------- 1 | >>> from frenchdeck import FrenchDeck, Card 2 | >>> beer_card = Card('7', 'diamonds') 3 | >>> beer_card 4 | Card(rank='7', suit='diamonds') 5 | >>> deck = FrenchDeck() 6 | >>> len(deck) 7 | 52 8 | >>> deck[:3] 9 | [Card(rank='2', suit='spades'), Card(rank='3', suit='spades'), Card(rank='4', suit='spades')] 10 | >>> deck[12::13] 11 | [Card(rank='A', suit='spades'), Card(rank='A', suit='diamonds'), Card(rank='A', suit='clubs'), Card(rank='A', suit='hearts')] 12 | >>> Card('Q', 'hearts') in deck 13 | True 14 | >>> Card('Z', 'clubs') in deck 15 | False 16 | >>> for card in deck: # doctest: +ELLIPSIS 17 | ... print(card) 18 | Card(rank='2', suit='spades') 19 | Card(rank='3', suit='spades') 20 | Card(rank='4', suit='spades') 21 | ... 22 | >>> for card in reversed(deck): # doctest: +ELLIPSIS 23 | ... print(card) 24 | Card(rank='A', suit='hearts') 25 | Card(rank='K', suit='hearts') 26 | Card(rank='Q', suit='hearts') 27 | ... 28 | >>> for n, card in enumerate(deck, 1): # doctest: +ELLIPSIS 29 | ... print(n, card) 30 | 1 Card(rank='2', suit='spades') 31 | 2 Card(rank='3', suit='spades') 32 | 3 Card(rank='4', suit='spades') 33 | ... 34 | >>> suit_values = dict(spades=3, hearts=2, diamonds=1, clubs=0) 35 | >>> def spades_high(card): 36 | ... rank_value = FrenchDeck.ranks.index(card.rank) 37 | ... return rank_value * len(suit_values) + suit_values[card.suit] 38 | 39 | Rank test: 40 | 41 | >>> spades_high(Card('2', 'clubs')) 42 | 0 43 | >>> spades_high(Card('A', 'spades')) 44 | 51 45 | 46 | >>> for card in sorted(deck, key=spades_high): # doctest: +ELLIPSIS 47 | ... print(card) 48 | Card(rank='2', suit='clubs') 49 | Card(rank='2', suit='diamonds') 50 | Card(rank='2', suit='hearts') 51 | ... 52 | Card(rank='A', suit='diamonds') 53 | Card(rank='A', suit='hearts') 54 | Card(rank='A', suit='spades') 55 | -------------------------------------------------------------------------------- /data-model/frenchdeck.py: -------------------------------------------------------------------------------- 1 | import collections 2 | 3 | Card = collections.namedtuple('Card', ['rank', 'suit']) 4 | 5 | class FrenchDeck: 6 | ranks = [str(n) for n in range(2, 11)] + list('JQKA') 7 | suits = 'spades diamonds clubs hearts'.split() 8 | 9 | def __init__(self): 10 | self._cards = [Card(rank, suit) for suit in self.suits 11 | for rank in self.ranks] 12 | 13 | def __len__(self): 14 | return len(self._cards) 15 | 16 | def __getitem__(self, position): 17 | return self._cards[position] 18 | -------------------------------------------------------------------------------- /data-model/vector2d.py: -------------------------------------------------------------------------------- 1 | from math import hypot 2 | 3 | class Vector: 4 | 5 | def __init__(self, x=0, y=0): 6 | self.x = x 7 | self.y = y 8 | 9 | def __repr__(self): 10 | return 'Vector(%r, %r)' % (self.x, self.y) 11 | 12 | def __abs__(self): 13 | return hypot(self.x, self.y) 14 | 15 | def __bool__(self): 16 | return bool(abs(self)) 17 | 18 | def __add__(self, other): 19 | x = self.x + other.x 20 | y = self.y + other.y 21 | return Vector(x, y) 22 | 23 | def __mul__(self, scalar): 24 | return Vector(self.x * scalar, self.y * scalar) 25 | -------------------------------------------------------------------------------- /intro/notebook_components.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramalho/python-eng/59a007d54595f11441ec7de07909576e1c158772/intro/notebook_components.webp -------------------------------------------------------------------------------- /memoria/fig2-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramalho/python-eng/59a007d54595f11441ec7de07909576e1c158772/memoria/fig2-1.png -------------------------------------------------------------------------------- /memoria/fig2-4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramalho/python-eng/59a007d54595f11441ec7de07909576e1c158772/memoria/fig2-4.png -------------------------------------------------------------------------------- /memoria/fig6-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramalho/python-eng/59a007d54595f11441ec7de07909576e1c158772/memoria/fig6-3.png -------------------------------------------------------------------------------- /memoria/fig6-4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramalho/python-eng/59a007d54595f11441ec7de07909576e1c158772/memoria/fig6-4.png -------------------------------------------------------------------------------- /memoria/var-boxes-x-labels.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramalho/python-eng/59a007d54595f11441ec7de07909576e1c158772/memoria/var-boxes-x-labels.png -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.ruff] 2 | line-length = 120 3 | [tool.ruff.format] 4 | # Prefer single quotes over double quotes. 5 | quote-style = "single" 6 | -------------------------------------------------------------------------------- /tipos/columnize.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Sequence 2 | 3 | def columnize( 4 | sequence: Sequence[str], num_columns: int = 0 5 | ) -> list[tuple[str, ...]]: 6 | if num_columns == 0: 7 | num_columns = round(len(sequence) ** 0.5) 8 | num_rows, reminder = divmod(len(sequence), num_columns) 9 | num_rows += bool(reminder) 10 | return [tuple(sequence[i::num_rows]) for i in range(num_rows)] 11 | 12 | -------------------------------------------------------------------------------- /tipos/coord_nt.py: -------------------------------------------------------------------------------- 1 | from typing import NamedTuple 2 | 3 | class Coordinate(NamedTuple): 4 | lat: float # (1) 5 | lon: float 6 | reference: str = 'WGS84' # (2) 7 | 8 | # experimento 9 | 10 | sp = Coordinate(44, -23) 11 | print(sp) 12 | -------------------------------------------------------------------------------- /tipos/coord_tuple.py: -------------------------------------------------------------------------------- 1 | from geolib import geohash as gh # type: ignore # (1) 2 | 3 | PRECISION = 9 4 | 5 | def geohash(lat_lon: tuple[float, float]) -> str: # (2) 6 | return gh.encode(*lat_lon, PRECISION) 7 | 8 | -------------------------------------------------------------------------------- /tipos/double.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Sequence 2 | from typing import Any, Protocol, TypeVar, TYPE_CHECKING 3 | from fractions import Fraction 4 | 5 | T = TypeVar('T') 6 | 7 | 8 | class Repetível(Protocol): 9 | def __mul__(self: T, other: Any) -> T: 10 | ... 11 | 12 | TR = TypeVar('TR', bound=Repetível) 13 | 14 | def dobro(x: TR) -> TR: 15 | return x * 2 16 | 17 | x = dobro(2) + 10 18 | 19 | if TYPE_CHECKING: 20 | reveal_type(x) 21 | 22 | print(dobro(3.5)) 23 | 24 | print(dobro('ma')) 25 | 26 | print(dobro(Fraction(2, 3))) 27 | 28 | print(dobro([1, 2, 3])) 29 | 30 | # erro proposital, nunca vai funcionar 31 | # print(dobro(None)) 32 | -------------------------------------------------------------------------------- /tipos/messages.py: -------------------------------------------------------------------------------- 1 | def show_count(count: int, word: str, plural_word: str | None = None) -> str: 2 | if count == 1: 3 | return f'1 {word}' 4 | count_str = str(count) if count else 'no' 5 | if plural_word: 6 | return f'{count_str} {plural_word}' 7 | return f'{count_str} {word}s' 8 | -------------------------------------------------------------------------------- /tipos/messages_tests.py: -------------------------------------------------------------------------------- 1 | from pytest import mark 2 | 3 | from messages import show_count 4 | 5 | 6 | @mark.parametrize( 7 | 'qty, expected', 8 | [ 9 | (1, '1 part'), 10 | (2, '2 parts'), 11 | ], 12 | ) 13 | def test_show_count(qty: int, expected: str) -> None: 14 | got = show_count(qty, 'part') 15 | assert got == expected 16 | 17 | 18 | def test_show_count_zero() -> None: 19 | got = show_count(0, 'part') 20 | assert got == 'no parts' 21 | 22 | 23 | def test_irregular() -> None: 24 | got = show_count(2, 'child', 'children') 25 | assert got == '2 children' 26 | -------------------------------------------------------------------------------- /tipos/mode_hashable.py: -------------------------------------------------------------------------------- 1 | from collections import Counter 2 | from collections.abc import Iterable, Hashable 3 | from typing import TypeVar, TYPE_CHECKING 4 | 5 | HashableT = TypeVar('HashableT', bound=Hashable) 6 | 7 | def mode(data: Iterable[HashableT]) -> HashableT: 8 | pairs = Counter(data).most_common(1) 9 | if len(pairs) == 0: 10 | raise ValueError('no mode for empty data') 11 | return pairs[0][0] 12 | 13 | 14 | m = mode([3, 4, 1, 3, 1, 3, 3, 2]) 15 | 16 | if TYPE_CHECKING: 17 | reveal_type(m) 18 | 19 | print(m) 20 | 21 | print(m * 10) 22 | -------------------------------------------------------------------------------- /tipos/mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | python_version = 3.11 3 | warn_unused_configs = True 4 | disallow_incomplete_defs = True 5 | -------------------------------------------------------------------------------- /tipos/replacer.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Iterable 2 | from typing import TypeAlias 3 | 4 | FromTo: TypeAlias = tuple[str, str] # (1) 5 | 6 | def zip_replace(text: str, changes: Iterable[FromTo]) -> str: # (2) 7 | for from_, to in changes: 8 | text = text.replace(from_, to) 9 | return text 10 | -------------------------------------------------------------------------------- /tipos/sample.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Sequence 2 | from random import shuffle 3 | from typing import TypeVar 4 | 5 | T = TypeVar('T') 6 | 7 | def sample(population: Sequence[T], size: int) -> list[T]: 8 | if size < 1: 9 | raise ValueError('size must be >= 1') 10 | result = list(population) 11 | shuffle(result) 12 | return result[:size] 13 | 14 | 15 | lista = list(range(1000)) 16 | 17 | print(sample(lista, 10)) 18 | 19 | --------------------------------------------------------------------------------