├── .coveragerc ├── .flake8 ├── .github └── workflows │ └── validate.yml ├── .gitignore ├── .pylintrc ├── README.md ├── lesson-01 ├── homework.md ├── lesson-01.pdf ├── test_user.py ├── user.py └── vk_api.py ├── lesson-02 ├── class_02.ipynb ├── homework.md └── lesson-02.pdf ├── lesson-03 ├── class_03.ipynb ├── homework.md └── lesson-03.pdf ├── lesson-04 ├── class_04.ipynb ├── homework.md └── lesson-04.pdf ├── lesson-05 ├── class_05.ipynb ├── homework.md └── lesson-05.pdf ├── lesson-06 ├── class_06.ipynb ├── homework.md └── lesson-06.pdf ├── lesson-07 ├── class_07.ipynb ├── homework.md ├── lesson-07.pdf └── src │ ├── generator_socket.py │ ├── select_socket.py │ ├── selectors_socket.py │ └── socket_server.py ├── lesson-08 ├── class_08.ipynb ├── homework.md └── lesson-08.pdf ├── lesson-09 ├── class_09.ipynb ├── homework.md └── lesson-09.pdf ├── lesson-10 ├── homework.md ├── lesson-10.pdf └── src │ ├── fib_capi │ ├── fibutils_capi.c │ └── setup.py │ ├── fib_cffi │ └── test_cffi.py │ ├── fib_ctypes │ ├── fibutils.c │ └── test_fibutils.py │ ├── fib_cython │ ├── fibcyth.pyx │ └── setup.py │ ├── fib_native.py │ └── perf.py ├── lesson-11 ├── check_hints.py └── lesson-11.pdf └── requirements.txt /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | exclude_also = 3 | logger\.info\(.*\) 4 | if __name__ == .__main__.: 5 | fail_under = 90 6 | precision = 2 7 | show_missing = true 8 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 80 3 | extend-ignore = F403, F405, F401 4 | -------------------------------------------------------------------------------- /.github/workflows/validate.yml: -------------------------------------------------------------------------------- 1 | name: Validation 2 | 3 | on: 4 | push: 5 | branches: [ "main" ] 6 | pull_request: 7 | branches: [ "main" ] 8 | 9 | workflow_dispatch: 10 | 11 | jobs: 12 | validate_homeworks: 13 | name: Validate all homeworks 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - uses: actions/checkout@v4 18 | - uses: actions/setup-python@v5.2.0 19 | with: 20 | python-version: '3.13' 21 | cache: 'pip' 22 | 23 | - name: Install dependencies 24 | run: pip install -r requirements.txt 25 | 26 | - name: Check flake8 27 | run: flake8 . 28 | 29 | - name: Check pylint 30 | if: ${{ always() }} 31 | run: pylint ./*/*.py 32 | 33 | - name: Check tests with pytest 34 | if: ${{ always() }} 35 | run: coverage run -m pytest . 36 | 37 | - name: Check test coverage 38 | run: coverage report -m 39 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # UV 98 | # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | #uv.lock 102 | 103 | # poetry 104 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 105 | # This is especially recommended for binary packages to ensure reproducibility, and is more 106 | # commonly ignored for libraries. 107 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 108 | #poetry.lock 109 | 110 | # pdm 111 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 112 | #pdm.lock 113 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 114 | # in version control. 115 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 116 | .pdm.toml 117 | .pdm-python 118 | .pdm-build/ 119 | 120 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 121 | __pypackages__/ 122 | 123 | # Celery stuff 124 | celerybeat-schedule 125 | celerybeat.pid 126 | 127 | # SageMath parsed files 128 | *.sage.py 129 | 130 | # Environments 131 | .env 132 | .venv 133 | env/ 134 | venv/ 135 | ENV/ 136 | env.bak/ 137 | venv.bak/ 138 | 139 | # Spyder project settings 140 | .spyderproject 141 | .spyproject 142 | 143 | # Rope project settings 144 | .ropeproject 145 | 146 | # mkdocs documentation 147 | /site 148 | 149 | # mypy 150 | .mypy_cache/ 151 | .dmypy.json 152 | dmypy.json 153 | 154 | # Pyre type checker 155 | .pyre/ 156 | 157 | # pytype static type analyzer 158 | .pytype/ 159 | 160 | # Cython debug symbols 161 | cython_debug/ 162 | 163 | # PyCharm 164 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 165 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 166 | # and can be added to the global gitignore or merged into this file. For a more nuclear 167 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 168 | #.idea/ 169 | 170 | # PyPI configuration file 171 | .pypirc 172 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | disable= 3 | C0114, # missing-module-docstring 4 | C0115, 5 | C0116, 6 | 7 | [FORMAT] 8 | max-line-length=80 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # deep_python_spring_2025 2 | Материалы открытого курса Углубленный Python от VK Education, весна 2025 3 | 4 | ## Лекции и материалы (слайды, домашки, код с занятий) 5 | 01. [Введение, типы данных, управляющие конструкции, тестирование](lesson-01) 6 | 02. [Функции](lesson-02) 7 | 03. [Классы, ООП](lesson-03) 8 | 04. [Дескрипторы, метапрограммироование, ABC](lesson-04) 9 | 05. [Стандартная библиотека](lesson-05) 10 | 06. [Потоки, процессы, IPC](lesson-06) 11 | 07. [Асинхронное программирование](lesson-07) 12 | 08. [Память, профилирование](lesson-08) 13 | 09. [Логирование, отладка](lesson-09) 14 | 10. [C-расширения](lesson-10) 15 | 11. [Аннотация типов](lesson-11) 16 | 17 | ## Правила оформления домашек на github 18 | * приватный github репозиторий с каким-то связанным с курсом названием (например, deep_python_25a_username); 19 | * сдача ДЗ должна выполняться в основной ветке репозитория **main**; 20 | * каждое ДЗ должно размещаться в каталоге с именем по номеру ДЗ (01, 02, 03 и тд) в корне репозитория: код, тесты и любые другие материалы, например, каталог для первой домашки должен называться "01"; 21 | * вспомогательных каталоги внутри каталога решения (01, 02) не должно быть, то есть все файлы и модули одной домашки должны находиться в корневом каталоге этой домашки (01, 02); 22 | * зеленый пайплайн является обязательным. 23 | 24 | ## FAQ 25 | * Допускается ли списывание? 26 | > Нет, **списывать нельзя**, но можно использовать различные ресурсы для подготовки. За списывание штраф 3 балла и попытка сдачи данной домашки, то есть можно будет пересдать списанную домашку только с одной попыткой. За повторное списывание студент отчисляется с курса. 27 | * Обязательно ли соблюдение правил оформления репозитория с домашками, как указано в приветственном посте на портале? 28 | > Да. 29 | * Как называть модули с решением? 30 | > Модули с решением должны именоваться исходя из их содержимого, например, `message_predictor.py`, а `task1.py` модуль называть не следует. 31 | * Как называть и располагать тестовые модули? 32 | > Тестовые модули должны именоваться с префиксом `test_` и находиться в корне каталога домашки (01, 02 и тд), для каждой части задания должен быть свой тестовый модуль. 33 | * Как работает мягкий дедлайн? 34 | > Если отправить домашку на проверку до наступления мягкого дедлайна по этой домашке, то можно получить полный балл (7). Более того, вторая и третья попытки, отправленные даже после дедлайна, уже штрафоваться не будут. 35 | * Как отправлять вторую и третью попытки сдачи ДЗ? 36 | > Обязательно нужно переотправить через портал (кнопка "Дополнить" в интерфейсе сдачи), как это выполнялось с первой попыткой. Можно дополнить комментарием в обсуждении, но коментарий не является обязательным. Только комментария недостаточно, тк если случайно его просмотреть, портал будет считать эту домашку проверенной и проверка правок по ней может сильно отсрочиться. 37 | 38 | -------------------------------------------------------------------------------- /lesson-01/homework.md: -------------------------------------------------------------------------------- 1 | # Домашнее задание #01 (введение, тестирование) 2 | 3 | ### 1. Функция оценки сообщения 4 | Реализовать функцию `predict_message_mood`, которая принимает на вход строку `message` и пороги хорошести. 5 | Функция возвращает: 6 | - "неуд", если предсказание модели меньше `bad_threshold`; 7 | - "отл", если предсказание модели больше `good_threshold`; 8 | - "норм" в остальных случаях. 9 | 10 | Функция `predict_message_mood` создает экземпляр класса `SomeModel` и вызывает у этого экземпляра метод `predict` с аргументом `message`. 11 | 12 | ```py 13 | class SomeModel: 14 | def predict(self, message: str) -> float: 15 | # реализация не важна 16 | 17 | 18 | def predict_message_mood( 19 | message: str, 20 | bad_thresholds: float = 0.3, 21 | good_thresholds: float = 0.8, 22 | ) -> str: 23 | ... 24 | model.predict() 25 | ... 26 | 27 | 28 | assert predict_message_mood("Чапаев и пустота") == "отл" 29 | assert predict_message_mood("Чапаев и пустота", 0.8, 0.99) == "норм" 30 | assert predict_message_mood("Вулкан") == "неуд" 31 | ``` 32 | 33 | ### 2. Генератор для чтения и фильтрации файла 34 | Есть текстовый файл, который может не помещаться в память. 35 | В каждой строке файла фраза или предложение: набор слов, разделенных пробелами (знаков препинания нет). 36 | 37 | Генератор должен принимать на вход: 38 | - имя файла или файловый объект; 39 | - список слов для поиска; 40 | - список стоп-слов. 41 | 42 | Генератор перебирает строки файла и возвращает только те из них (строку целиком), где встретилось хотя бы одно из слов для поиска. 43 | Если в одной строке сразу несколько совпадений, то вернуть строку надо лишь один раз. 44 | Если в строке встретилось слово из списка стоп-слов, то такая строка должна игнорирроваться, даже если там есть совпадения по словам поиска. 45 | Поиск совпадений и стоп-слов должен выполняться по полному совпадению слова без учета регистра. 46 | 47 | Например, для строки из файла "а Роза упала на лапу Азора" слово поиска "роза" должно найтись, а "роз" или "розан" - уже нет. 48 | В случае той же строки "а Роза упала на лапу Азора", слова-совпадения "роза" и стоп-слова "азора" исходная строка должна будет быть отброщена. 49 | 50 | ### 3. Тесты в отдельном модуле для каждого пункта 51 | 52 | ### 4. Зеленый пайплайн в репе 53 | Обязательно: тесты, покрытие, flake8, pylint. 54 | Опционально можно добавить другие инструменты, например, mypy и black. 55 | Покрытие тестов должно составлять не менее 90%. 56 | -------------------------------------------------------------------------------- /lesson-01/lesson-01.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mailcourses/deep_python_spring_2025/ede9fa206266b26d1da2c1857c8c41482677c3b9/lesson-01/lesson-01.pdf -------------------------------------------------------------------------------- /lesson-01/test_user.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest import mock 3 | 4 | from user import User 5 | 6 | 7 | class TestUser(unittest.TestCase): 8 | 9 | def test_init(self): 10 | usr = User("steve", 42) 11 | self.assertEqual("steve", usr.name) 12 | self.assertEqual(42, usr.age) 13 | 14 | def test_greetings(self): 15 | usr = User("steve", 42) 16 | self.assertEqual("steve", usr.name) 17 | self.assertEqual("Hello, steve!", usr.greetings()) 18 | 19 | usr = User("", 42) 20 | self.assertEqual("", usr.name) 21 | self.assertEqual("Hello, !", usr.greetings()) 22 | 23 | def test_birthday(self): 24 | usr = User("steve", 42) 25 | self.assertEqual(42, usr.age) 26 | self.assertEqual(43, usr.birthday()) 27 | self.assertEqual(43, usr.age) 28 | 29 | def test_friends_not_impl(self): 30 | usr = User("steve", 42) 31 | 32 | with self.assertRaises(NotImplementedError): 33 | usr.get_friends() 34 | 35 | def test_friends_empty(self): 36 | usr = User("steve", 42) 37 | 38 | with mock.patch("user.fetch_vk_api") as mock_api: 39 | mock_api.return_value = [] 40 | 41 | friends = usr.get_friends() 42 | self.assertEqual([], friends) 43 | 44 | calls = [ 45 | mock.call("/friends", "steve", part=None), 46 | ] 47 | self.assertEqual(calls, mock_api.mock_calls) 48 | 49 | friends = usr.get_friends(name_part="apple") 50 | self.assertEqual([], friends) 51 | 52 | calls = [ 53 | mock.call("/friends", "steve", part=None), 54 | mock.call("/friends", "steve", part="APPLE"), 55 | ] 56 | self.assertEqual(calls, mock_api.mock_calls) 57 | 58 | def test_friends_single(self): 59 | usr = User("steve", 42) 60 | 61 | with mock.patch("user.fetch_vk_api") as mock_api: 62 | mock_api.side_effect = [["voz"], ["voz", "lisa"]] 63 | 64 | friends = usr.get_friends() 65 | self.assertEqual(["voz"], friends) 66 | 67 | calls = [ 68 | mock.call("/friends", "steve", part=None), 69 | ] 70 | self.assertEqual(calls, mock_api.mock_calls) 71 | 72 | friends = usr.get_friends("is") 73 | self.assertEqual(["lisa"], friends) 74 | 75 | calls = [ 76 | mock.call("/friends", "steve", part=None), 77 | mock.call("/friends", "steve", part="IS"), 78 | ] 79 | self.assertEqual(calls, mock_api.mock_calls) 80 | 81 | # friends = usr.get_friends("is") 82 | 83 | @mock.patch("user.fetch_vk_api") 84 | def test_friends_no_filter(self, mock_api): 85 | usr = User("steve", 42) 86 | 87 | def get_friends(*_, **__): 88 | return ["voz", "lisa"] 89 | 90 | mock_api.side_effect = get_friends 91 | 92 | friends = usr.get_friends() 93 | self.assertEqual(["voz", "lisa"], friends) 94 | 95 | calls = [ 96 | mock.call("/friends", "steve", part=None), 97 | ] 98 | self.assertEqual(calls, mock_api.mock_calls) 99 | 100 | friends = usr.get_friends() 101 | self.assertEqual(["voz", "lisa"], friends) 102 | 103 | calls = [ 104 | mock.call("/friends", "steve", part=None), 105 | mock.call("/friends", "steve", part=None), 106 | ] 107 | self.assertEqual(calls, mock_api.mock_calls) 108 | 109 | @mock.patch("user.fetch_vk_api") 110 | def test_friends_connection_error(self, mock_api): 111 | usr = User("steve", 42) 112 | 113 | mock_api.side_effect = Exception("connection error") 114 | 115 | with self.assertRaises(Exception) as err: 116 | usr.get_friends() 117 | 118 | self.assertEqual("connection error", str(err.exception)) 119 | 120 | calls = [ 121 | mock.call("/friends", "steve", part=None), 122 | ] 123 | self.assertEqual(calls, mock_api.mock_calls) 124 | 125 | with self.assertRaises(Exception) as err: 126 | usr.get_friends("is") 127 | 128 | self.assertEqual("connection error", str(err.exception)) 129 | 130 | calls = [ 131 | mock.call("/friends", "steve", part=None), 132 | mock.call("/friends", "steve", part="IS"), 133 | ] 134 | self.assertEqual(calls, mock_api.mock_calls) 135 | -------------------------------------------------------------------------------- /lesson-01/user.py: -------------------------------------------------------------------------------- 1 | from vk_api import fetch_vk_api 2 | 3 | 4 | class User: 5 | def __init__(self, name, age): 6 | self.name = name 7 | self.age = age 8 | 9 | def greetings(self): 10 | return f"Hello, {self.name}!" 11 | 12 | def birthday(self): 13 | self.age += 1 14 | return self.age 15 | 16 | def get_friends(self, name_part=None): 17 | if name_part: 18 | name_part = name_part.upper() 19 | 20 | friends = fetch_vk_api( 21 | "/friends", self.name, part=name_part 22 | ) 23 | 24 | if name_part is not None: 25 | name_part = name_part.lower() 26 | friends = [ 27 | fr for fr in friends if name_part in fr 28 | ] 29 | 30 | return friends 31 | 32 | 33 | if __name__ == "__main__": 34 | usr = User("steve", 42) 35 | print(usr.greetings()) 36 | -------------------------------------------------------------------------------- /lesson-01/vk_api.py: -------------------------------------------------------------------------------- 1 | def fetch_vk_api(path, username, part): 2 | # requests.get(path, ...) 3 | raise NotImplementedError 4 | -------------------------------------------------------------------------------- /lesson-02/homework.md: -------------------------------------------------------------------------------- 1 | # Домашнее задание #02 (функции) 2 | 3 | ### 1. Функция для обработки json 4 | Функция для обработки должна принимать параметры: 5 | - строку с json; 6 | - список ключей, которые необходимо обработать; 7 | - список токенов, которые нужно найти; 8 | - функцию-обработчик ключа и токена. 9 | 10 | json для задания всегда имеет вид словаря с ключами и значениями из строк. Строки-значения в json содержат произвольное число пробелов в качестве разделителей слов, знаков препинания нет. 11 | 12 | Все токены и ключи состоят только из букв и цифр без пробельных символов. 13 | 14 | Функция парсит строку с json библиотечными средствами. 15 | Для каждого ключа json, который совпадает с одним из переданных ключей для обработки, функция должна искать вхождения токенов в строку-значение по данному ключу. 16 | Для каждого найденного токена должна быть вызвана функция-обработчик с ключом и токеном. 17 | 18 | Поиск ключей должен зависеть от регистра, а поиск токенов должен быть регистронезависимым. 19 | 20 | 21 | ```py 22 | def process_json( 23 | json_str: str, 24 | required_keys: list[str] | None = None, 25 | tokens: list[str] | None = None, 26 | callback: Callable[[str, str], None] | None = None, 27 | ) -> None: 28 | ... 29 | 30 | 31 | # например: 32 | json_str = '{"key1": "Word1 word2", "key2": "word2 word3"}' 33 | required_keys = ["key1", "KEY2"] 34 | tokens = ["WORD1", "word2"] 35 | 36 | process_json(json_str, required_keys, tokens, lambda key, token: f"{key=}, {token=}") 37 | 38 | # выведет: 39 | # key="key1", token="WORD1" 40 | # key="key1", token="word2" 41 | ``` 42 | 43 | ### 2. Параметризуемый декоратор для логирования вызовов и перезапуска функций в случае ошибок 44 | Декоратор `retry_deco` должен: 45 | - принимать опциональными параметрами число перезапусков декорируемой функции и список ожидаемых классов исключений; 46 | - при вызове функции логировать (выводить) название функции, все переданные ей аргументы, номер попытки перезапуска, результат работы функции и ошибку, если было выброшено исключение; 47 | формат логирования произвольный (например, функция и аргументы один раз, а номер попытки и исключение/результат сколько потребуется); 48 | - в случае исключения при выполнении функции декоратор должен выполнить новую попытку запуска функции, пока не достигнет заданного числа перезапусков; 49 | если исключение из списка ожидаемых классов исключений (параметр декоратора), то перезапускать функцию не надо, тк исключения из списка это нормальный режим работы декорируемой функции. 50 | 51 | ```py 52 | def retry_deco(...): 53 | ... 54 | 55 | 56 | @retry_deco(3) 57 | def add(a, b): 58 | return a + b 59 | 60 | 61 | add(4, 2) 62 | # run "add" with positional args = (4, 2), attempt = 1, result = 6 63 | 64 | add(4, b=3) 65 | # run "add" with positional args = (4,), keyword kwargs = {"b": 3}, attempt = 1, result = 7 66 | 67 | 68 | @retry_deco(3) 69 | def check_str(value=None): 70 | if value is None: 71 | raise ValueError() 72 | 73 | return isinstance(value, str) 74 | 75 | 76 | check_str(value="123") 77 | # run "check_str" with keyword kwargs = {"value": "123"}, attempt = 1, result = True 78 | 79 | check_str(value=1) 80 | # run "check_str" with keyword kwargs = {"value": 1}, attempt = 1, result = False 81 | 82 | check_str(value=None) 83 | # run "check_str" with keyword kwargs = {"value": None}, attempt = 1, exception = ValueError 84 | # run "check_str" with keyword kwargs = {"value": None}, attempt = 2, exception = ValueError 85 | # run "check_str" with keyword kwargs = {"value": None}, attempt = 3, exception = ValueError 86 | 87 | 88 | @retry_deco(2, [ValueError]) 89 | def check_int(value=None): 90 | if value is None: 91 | raise ValueError() 92 | 93 | return isinstance(value, int) 94 | 95 | check_int(value=1) 96 | # run "check_int" with keyword kwargs = {"value": 1}, attempt = 1, result = True 97 | 98 | check_int(value=None) 99 | # run "check_int" with keyword kwargs = {"value": None}, attempt = 1, exception = ValueError # нет перезапуска 100 | 101 | ``` 102 | 103 | ### 3. Тесты в отдельном модуле для каждого пункта 104 | 105 | ### 4. Зеленый пайплайн в репе 106 | Обязательно: тесты, покрытие, flake8, pylint. 107 | Опционально можно добавить другие инструменты, например, mypy и black. 108 | Покрытие тестов должно составлять не менее 90%. 109 | -------------------------------------------------------------------------------- /lesson-02/lesson-02.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mailcourses/deep_python_spring_2025/ede9fa206266b26d1da2c1857c8c41482677c3b9/lesson-02/lesson-02.pdf -------------------------------------------------------------------------------- /lesson-03/homework.md: -------------------------------------------------------------------------------- 1 | # Домашнее задание #03 (Классы, ООП) 2 | 3 | ### 1. Реализовать класс CustomList наследованием от list 4 | 5 | При этом: 6 | - `CustomList` должен наследоваться от встроенного списка `list` для получения всех методов последнего (UserList можно, но мамнипулировать его data не следует); 7 | - экземпляры `CustomList` можно складывать и вычитать друг с другом, с обычными списками и с числами: 8 | ```py 9 | CustomList([5, 1, 3, 7]) + CustomList([1, 2, 7]) # CustomList([6, 3, 10, 7]) 10 | CustomList([10]) + [2, 5] # CustomList([12, 5]) 11 | [2, 5] + CustomList([10]) # CustomList([12, 5]) 12 | CustomList([2, 5]) + 10 # CustomList([12, 15]) 13 | 10 + CustomList([2, 5]) # CustomList([12, 15]) 14 | 15 | CustomList([5, 1, 3, 7]) - CustomList([1, 2, 7]) # CustomList([4, -1, -4, 7]) 16 | CustomList([10]) - [2, 5] # CustomList([8, -5]) 17 | [2, 5] - CustomList([10]) # CustomList([-8, 5]) 18 | CustomList([2, 5]) - 10 # CustomList([-8, -5]) 19 | 10 - CustomList([2, 5]) # CustomList([8, 5]) 20 | ``` 21 | Возвращаться должен новый экземпляр `CustomList`, элементы которого будут результатом поэлементного сложения/вычитания элементов исходных списков. 22 | Сложение/вычитание с числом выполняется как сложение/вычитание каждого элемента списка с данным числом; 23 | - при сложении/вычитании списков разной длины отсутствующие элементы меньшего списка считаются нулями; 24 | - после сложения/вычитания исходные списки не должны изменяться; 25 | - при сравнении (`==`, `!=`, `>`, `>=`, `<`, `<=`) экземмпляров CustomList должна сравниваться сумма элементов списков (сравнение с `list` и `int` не нужно); 26 | - должен быть переопределен `str`, чтобы выводились элементы списка и их сумма; 27 | - можно считать элементы списка `CustomList`, `list` и другие операнды всегда всегда целыми числами. 28 | 29 | ### 2. Тесты CustomList в отдельном модуле 30 | Тесты должны обязательно охватывать переопределенные методы. 31 | 32 | ### 3. Зеленый пайплайн в репе 33 | Обязательно: тесты, покрытие, flake8, pylint. 34 | Опционально можно добавить другие инструменты, например, mypy и black. 35 | Покрытие тестов должно составлять не менее 90%. 36 | -------------------------------------------------------------------------------- /lesson-03/lesson-03.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mailcourses/deep_python_spring_2025/ede9fa206266b26d1da2c1857c8c41482677c3b9/lesson-03/lesson-03.pdf -------------------------------------------------------------------------------- /lesson-04/homework.md: -------------------------------------------------------------------------------- 1 | # Домашнее задание #04 (дескрипторы, метаклассы, ABC) 2 | 3 | ### 1. Метакласс, который в начале названий всех атрибутов и методов, кроме магических, добавляет префикс "custom_" 4 | Подменяться должны атрибуты класса и атрибуты экземпляра класса, в том числе добавленные после выполнения конструктора (dynamic в примере). 5 | 6 | ```py 7 | class CustomMeta(...): 8 | pass 9 | 10 | 11 | class CustomClass(metaclass=CustomMeta): 12 | x = 50 13 | 14 | def __init__(self, val=99): 15 | self.val = val 16 | 17 | def line(self): 18 | return 100 19 | 20 | def __str__(self): 21 | return "Custom_by_metaclass" 22 | 23 | 24 | assert CustomClass.custom_x == 50 25 | CustomClass.x # ошибка 26 | 27 | inst = CustomClass() 28 | assert inst.custom_x == 50 29 | assert inst.custom_val == 99 30 | assert inst.custom_line() == 100 31 | assert str(inst) == "Custom_by_metaclass" 32 | 33 | inst.x # ошибка 34 | inst.val # ошибка 35 | inst.line() # ошибка 36 | inst.yyy # ошибка 37 | 38 | inst.dynamic = "added later" 39 | assert inst.custom_dynamic == "added later" 40 | inst.dynamic # ошибка 41 | ``` 42 | 43 | 44 | ### 2. Дескрипторы с проверками типов и значений данных 45 | Нужно сделать три дескриптора для какой-то области интереса (наука, финансы, хобби и тд), но если совсем не получается, то можно использовать шаблона ниже в качестве основы. 46 | У дескрипторов должен быть базовый класс с абстрактным методом-проверкой, который наследующие классы должны будут реализовать. 47 | 48 | ```py 49 | class Base...: 50 | pass 51 | 52 | class Integer: 53 | pass 54 | 55 | class String: 56 | pass 57 | 58 | class PositiveInteger: 59 | pass 60 | 61 | class Data: 62 | num = Integer() 63 | name = String() 64 | price = PositiveInteger() 65 | 66 | def __init__(...): 67 | .... 68 | ``` 69 | 70 | 71 | ### 3. Тесты метакласса и дескрипторов 72 | 73 | ### 4. Зеленый пайплайн в репе 74 | Обязательно: тесты, покрытие, flake8, pylint. 75 | Опционально можно добавить другие инструменты, например, mypy и black. 76 | Покрытие тестов должно составлять не менее 90%. 77 | -------------------------------------------------------------------------------- /lesson-04/lesson-04.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mailcourses/deep_python_spring_2025/ede9fa206266b26d1da2c1857c8c41482677c3b9/lesson-04/lesson-04.pdf -------------------------------------------------------------------------------- /lesson-05/homework.md: -------------------------------------------------------------------------------- 1 | # Домашнее задание #05 (стандартная библиотека) 2 | 3 | ### 1. LRU-кэш 4 | Интерфейс: 5 | 6 | ```py 7 | class LRUCache: 8 | 9 | def __init__(self, limit=42): 10 | pass 11 | 12 | def get(self, key): 13 | pass 14 | 15 | def set(self, key, value): 16 | pass 17 | 18 | 19 | cache = LRUCache(2) 20 | 21 | cache.set("k1", "val1") 22 | cache.set("k2", "val2") 23 | 24 | assert cache.get("k3") is None 25 | assert cache.get("k2") == "val2" 26 | assert cache.get("k1") == "val1" 27 | 28 | cache.set("k3", "val3") 29 | 30 | assert cache.get("k3") == "val3" 31 | assert cache.get("k2") is None 32 | assert cache.get("k1") == "val1" 33 | 34 | 35 | Если удобнее, get/set можно сделать по аналогии с dict: 36 | cache["k1"] = "val1" 37 | print(cache["k3"]) 38 | ``` 39 | 40 | Сложность решения по времени в среднем должна быть константной O(1). 41 | Реализация любым способом без использования OrderedDict. 42 | 43 | ### 2. Тесты в отдельном модуле 44 | 45 | ### 3. Зеленый пайплайн в репе 46 | Обязательно: тесты, покрытие, flake8, pylint. 47 | Опционально можно добавить другие инструменты, например, mypy и black. 48 | Покрытие тестов должно составлять не менее 90%. 49 | -------------------------------------------------------------------------------- /lesson-05/lesson-05.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mailcourses/deep_python_spring_2025/ede9fa206266b26d1da2c1857c8c41482677c3b9/lesson-05/lesson-05.pdf -------------------------------------------------------------------------------- /lesson-06/class_06.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "9f93f6d1-0401-4a40-b155-d35a5758b667", 7 | "metadata": {}, 8 | "outputs": [ 9 | { 10 | "data": { 11 | "text/plain": [ 12 | "314" 13 | ] 14 | }, 15 | "execution_count": 1, 16 | "metadata": {}, 17 | "output_type": "execute_result" 18 | } 19 | ], 20 | "source": [ 21 | "int(\"314\")" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 2, 27 | "id": "8043aef5-f16d-4d81-a763-3e5203cebb1b", 28 | "metadata": {}, 29 | "outputs": [ 30 | { 31 | "data": { 32 | "text/plain": [ 33 | "3" 34 | ] 35 | }, 36 | "execution_count": 2, 37 | "metadata": {}, 38 | "output_type": "execute_result" 39 | } 40 | ], 41 | "source": [ 42 | "int(3.14)" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 4, 48 | "id": "4c6ed9ea-b27b-488d-8495-ba1ad16abe5b", 49 | "metadata": {}, 50 | "outputs": [ 51 | { 52 | "data": { 53 | "text/plain": [ 54 | "(3, -3, 4, -4)" 55 | ] 56 | }, 57 | "execution_count": 4, 58 | "metadata": {}, 59 | "output_type": "execute_result" 60 | } 61 | ], 62 | "source": [ 63 | "int(3.99), int(-3.99), int(4.99), int(-4.99)" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": 6, 69 | "id": "8f4a5d8b-f664-4c73-a5d4-2a3aac9bc487", 70 | "metadata": {}, 71 | "outputs": [ 72 | { 73 | "data": { 74 | "text/plain": [ 75 | "50860026" 76 | ] 77 | }, 78 | "execution_count": 6, 79 | "metadata": {}, 80 | "output_type": "execute_result" 81 | } 82 | ], 83 | "source": [ 84 | "int(\"0101010010101\", 5)" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": 8, 90 | "id": "cd14834c-792d-4fda-97ee-a409b4397e06", 91 | "metadata": {}, 92 | "outputs": [ 93 | { 94 | "data": { 95 | "text/plain": [ 96 | "(3, 3.1)" 97 | ] 98 | }, 99 | "execution_count": 8, 100 | "metadata": {}, 101 | "output_type": "execute_result" 102 | } 103 | ], 104 | "source": [ 105 | "round(3.14), round(3.14, 1)" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": 9, 111 | "id": "a35f93ca-4291-4f46-bf86-2b61ea375f37", 112 | "metadata": {}, 113 | "outputs": [ 114 | { 115 | "data": { 116 | "text/plain": [ 117 | "4" 118 | ] 119 | }, 120 | "execution_count": 9, 121 | "metadata": {}, 122 | "output_type": "execute_result" 123 | } 124 | ], 125 | "source": [ 126 | "round(3.5)" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": 10, 132 | "id": "a35ceba1-9f2a-4bb1-826b-dd77368f0806", 133 | "metadata": {}, 134 | "outputs": [ 135 | { 136 | "data": { 137 | "text/plain": [ 138 | "2" 139 | ] 140 | }, 141 | "execution_count": 10, 142 | "metadata": {}, 143 | "output_type": "execute_result" 144 | } 145 | ], 146 | "source": [ 147 | "round(2.5)" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": 14, 153 | "id": "0b2b2bde-482e-4dd1-a80b-7b67ba8feea0", 154 | "metadata": {}, 155 | "outputs": [ 156 | { 157 | "data": { 158 | "text/plain": [ 159 | "(3.5, 3.2, 3.9, 3.4, 3.5)" 160 | ] 161 | }, 162 | "execution_count": 14, 163 | "metadata": {}, 164 | "output_type": "execute_result" 165 | } 166 | ], 167 | "source": [ 168 | "round(3.55, 1), round(3.25, 1), round(3.85, 1), round(3.35, 1), round(3.45, 1)" 169 | ] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "execution_count": 15, 174 | "id": "91db6035-4c16-4025-a31e-c7aec96e88c6", 175 | "metadata": {}, 176 | "outputs": [ 177 | { 178 | "data": { 179 | "text/plain": [ 180 | "False" 181 | ] 182 | }, 183 | "execution_count": 15, 184 | "metadata": {}, 185 | "output_type": "execute_result" 186 | } 187 | ], 188 | "source": [ 189 | "0.1 + 0.2 == 0.3" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": 17, 195 | "id": "eff40129-e5f7-4bc2-be2b-e4a752b57b7c", 196 | "metadata": {}, 197 | "outputs": [ 198 | { 199 | "data": { 200 | "text/plain": [ 201 | "True" 202 | ] 203 | }, 204 | "execution_count": 17, 205 | "metadata": {}, 206 | "output_type": "execute_result" 207 | } 208 | ], 209 | "source": [ 210 | "abs(0.1 + 0.2 - 0.3) <= 2 ** -30" 211 | ] 212 | }, 213 | { 214 | "cell_type": "code", 215 | "execution_count": 18, 216 | "id": "3fa08fee-c449-472b-8849-2ff7e4a1f0b2", 217 | "metadata": {}, 218 | "outputs": [], 219 | "source": [ 220 | "import math" 221 | ] 222 | }, 223 | { 224 | "cell_type": "code", 225 | "execution_count": 20, 226 | "id": "3da866b5-3030-4d88-99f9-0204ad50a863", 227 | "metadata": {}, 228 | "outputs": [ 229 | { 230 | "data": { 231 | "text/plain": [ 232 | "True" 233 | ] 234 | }, 235 | "execution_count": 20, 236 | "metadata": {}, 237 | "output_type": "execute_result" 238 | } 239 | ], 240 | "source": [ 241 | "math.isclose(0.1 + 0.2, 0.3)" 242 | ] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "execution_count": null, 247 | "id": "5302dcf0-e5a0-4f23-967e-3dc5ecde4ea2", 248 | "metadata": {}, 249 | "outputs": [], 250 | "source": [] 251 | }, 252 | { 253 | "cell_type": "code", 254 | "execution_count": null, 255 | "id": "3003a98b-6cc0-4097-b307-48a3c3192205", 256 | "metadata": {}, 257 | "outputs": [], 258 | "source": [] 259 | }, 260 | { 261 | "cell_type": "code", 262 | "execution_count": null, 263 | "id": "83d1b71a-98d2-494b-85f3-14f1905671e2", 264 | "metadata": {}, 265 | "outputs": [], 266 | "source": [] 267 | }, 268 | { 269 | "cell_type": "code", 270 | "execution_count": null, 271 | "id": "33888778-5fa0-4c90-a772-4a91980e5f88", 272 | "metadata": {}, 273 | "outputs": [], 274 | "source": [] 275 | }, 276 | { 277 | "cell_type": "code", 278 | "execution_count": 29, 279 | "id": "a276fe66-73e1-4423-91a7-07c7a110b68f", 280 | "metadata": {}, 281 | "outputs": [], 282 | "source": [ 283 | "import threading\n", 284 | "import time" 285 | ] 286 | }, 287 | { 288 | "cell_type": "code", 289 | "execution_count": 22, 290 | "id": "65f3c17e-264b-4ae0-8a4f-18b32a0596c7", 291 | "metadata": {}, 292 | "outputs": [ 293 | { 294 | "data": { 295 | "text/plain": [ 296 | "8" 297 | ] 298 | }, 299 | "execution_count": 22, 300 | "metadata": {}, 301 | "output_type": "execute_result" 302 | } 303 | ], 304 | "source": [ 305 | "threading.active_count()" 306 | ] 307 | }, 308 | { 309 | "cell_type": "code", 310 | "execution_count": 23, 311 | "id": "85e81600-49ed-49f6-8bd0-aef8d710eb95", 312 | "metadata": {}, 313 | "outputs": [ 314 | { 315 | "name": "stderr", 316 | "output_type": "stream", 317 | "text": [ 318 | "/var/folders/c1/9wdb7dps2x332c6l1y2hrqkh0000gp/T/ipykernel_60215/2205834880.py:1: DeprecationWarning: activeCount() is deprecated, use active_count() instead\n", 319 | " threading.activeCount()\n" 320 | ] 321 | }, 322 | { 323 | "data": { 324 | "text/plain": [ 325 | "8" 326 | ] 327 | }, 328 | "execution_count": 23, 329 | "metadata": {}, 330 | "output_type": "execute_result" 331 | } 332 | ], 333 | "source": [ 334 | "threading.activeCount()" 335 | ] 336 | }, 337 | { 338 | "cell_type": "code", 339 | "execution_count": 24, 340 | "id": "ee16266a-c131-4637-bbf9-b49856927e65", 341 | "metadata": {}, 342 | "outputs": [ 343 | { 344 | "data": { 345 | "text/plain": [ 346 | "[<_MainThread(MainThread, started 140704311966528)>,\n", 347 | " ,\n", 348 | " ,\n", 349 | " ,\n", 350 | " ,\n", 351 | " ,\n", 352 | " ,\n", 353 | " ]" 354 | ] 355 | }, 356 | "execution_count": 24, 357 | "metadata": {}, 358 | "output_type": "execute_result" 359 | } 360 | ], 361 | "source": [ 362 | "threading.enumerate()" 363 | ] 364 | }, 365 | { 366 | "cell_type": "code", 367 | "execution_count": 25, 368 | "id": "5bdf73cd-e808-4429-8bb0-3e382f3f45aa", 369 | "metadata": {}, 370 | "outputs": [ 371 | { 372 | "data": { 373 | "text/plain": [ 374 | "140704311966528" 375 | ] 376 | }, 377 | "execution_count": 25, 378 | "metadata": {}, 379 | "output_type": "execute_result" 380 | } 381 | ], 382 | "source": [ 383 | "threading.get_ident()" 384 | ] 385 | }, 386 | { 387 | "cell_type": "code", 388 | "execution_count": 26, 389 | "id": "299f4fb6-7522-43ed-b4ad-c1e6d6145bfc", 390 | "metadata": {}, 391 | "outputs": [], 392 | "source": [ 393 | "th = threading.current_thread()" 394 | ] 395 | }, 396 | { 397 | "cell_type": "code", 398 | "execution_count": 27, 399 | "id": "33511ca9-2801-4521-aabd-2e0caced36f6", 400 | "metadata": {}, 401 | "outputs": [ 402 | { 403 | "data": { 404 | "text/plain": [ 405 | "<_MainThread(MainThread, started 140704311966528)>" 406 | ] 407 | }, 408 | "execution_count": 27, 409 | "metadata": {}, 410 | "output_type": "execute_result" 411 | } 412 | ], 413 | "source": [ 414 | "th" 415 | ] 416 | }, 417 | { 418 | "cell_type": "code", 419 | "execution_count": 28, 420 | "id": "801ea244-e6c9-4965-9f98-591441044db3", 421 | "metadata": {}, 422 | "outputs": [ 423 | { 424 | "data": { 425 | "text/plain": [ 426 | "('MainThread', False, 140704311966528, 8904570, True)" 427 | ] 428 | }, 429 | "execution_count": 28, 430 | "metadata": {}, 431 | "output_type": "execute_result" 432 | } 433 | ], 434 | "source": [ 435 | "th.name, th.daemon, th.ident, th.native_id, th.is_alive()" 436 | ] 437 | }, 438 | { 439 | "cell_type": "code", 440 | "execution_count": null, 441 | "id": "c3c420bd-28f3-42df-8651-769501bb8e1b", 442 | "metadata": {}, 443 | "outputs": [], 444 | "source": [] 445 | }, 446 | { 447 | "cell_type": "code", 448 | "execution_count": null, 449 | "id": "d0991f69-dae7-44c4-b12e-2f6271ef0363", 450 | "metadata": {}, 451 | "outputs": [], 452 | "source": [] 453 | }, 454 | { 455 | "cell_type": "code", 456 | "execution_count": 32, 457 | "id": "4e0c6096-b2d4-4f43-bc88-79e88b3a676d", 458 | "metadata": {}, 459 | "outputs": [ 460 | { 461 | "name": "stdout", 462 | "output_type": "stream", 463 | "text": [ 464 | "data.is_alive()=False\n", 465 | "self.name='Thread-6', self.n=10 -- started\n", 466 | "self.name='Thread-6', i=0\n", 467 | "self.name='Thread-6', i=1\n", 468 | "self.name='Thread-6', i=2\n", 469 | "self.name='Thread-6', i=3\n" 470 | ] 471 | } 472 | ], 473 | "source": [ 474 | "class DataProcessor(threading.Thread):\n", 475 | " def __init__(self, n):\n", 476 | " super().__init__()\n", 477 | " self.n = n\n", 478 | " self.result = 0\n", 479 | "\n", 480 | " def run(self):\n", 481 | " print(f\"{self.name=}, {self.n=} -- started\")\n", 482 | "\n", 483 | " for i in range(self.n):\n", 484 | " print(f\"{self.name=}, {i=}\")\n", 485 | " self.result += i\n", 486 | " time.sleep(1)\n", 487 | "\n", 488 | "\n", 489 | "data = DataProcessor(10)\n", 490 | "print(f\"{data.is_alive()=}\")\n", 491 | "\n", 492 | "data.start()\n" 493 | ] 494 | }, 495 | { 496 | "cell_type": "code", 497 | "execution_count": 33, 498 | "id": "645c3e7a-fe0c-4624-b28d-dce7218c9779", 499 | "metadata": {}, 500 | "outputs": [ 501 | { 502 | "name": "stdout", 503 | "output_type": "stream", 504 | "text": [ 505 | "something\n", 506 | "self.name='Thread-6', i=4\n", 507 | "self.name='Thread-6', i=5\n", 508 | "self.name='Thread-6', i=6\n" 509 | ] 510 | } 511 | ], 512 | "source": [ 513 | "print(\"something\")" 514 | ] 515 | }, 516 | { 517 | "cell_type": "code", 518 | "execution_count": 34, 519 | "id": "54afd867-0580-47fb-bc1e-1dd5b9a3dfae", 520 | "metadata": {}, 521 | "outputs": [ 522 | { 523 | "name": "stdout", 524 | "output_type": "stream", 525 | "text": [ 526 | "something\n", 527 | "self.name='Thread-6', i=7\n", 528 | "self.name='Thread-6', i=8\n", 529 | "self.name='Thread-6', i=9\n" 530 | ] 531 | } 532 | ], 533 | "source": [ 534 | "print(\"something\")" 535 | ] 536 | }, 537 | { 538 | "cell_type": "code", 539 | "execution_count": 35, 540 | "id": "a9098114-392a-41d0-80df-a5df3e8193b1", 541 | "metadata": {}, 542 | "outputs": [ 543 | { 544 | "data": { 545 | "text/plain": [ 546 | "45" 547 | ] 548 | }, 549 | "execution_count": 35, 550 | "metadata": {}, 551 | "output_type": "execute_result" 552 | } 553 | ], 554 | "source": [ 555 | "data.result" 556 | ] 557 | }, 558 | { 559 | "cell_type": "code", 560 | "execution_count": null, 561 | "id": "c6eadfb1-daa7-4208-87a3-a7ba56caaf8c", 562 | "metadata": {}, 563 | "outputs": [], 564 | "source": [] 565 | }, 566 | { 567 | "cell_type": "code", 568 | "execution_count": 36, 569 | "id": "4ea42932-f865-4660-b106-fda82281e7de", 570 | "metadata": {}, 571 | "outputs": [ 572 | { 573 | "name": "stdout", 574 | "output_type": "stream", 575 | "text": [ 576 | "data.is_alive()=False\n", 577 | "self.name='Thread-7', self.n=10 -- started\n", 578 | "self.name='Thread-7', i=0\n", 579 | "self.name='Thread-7', i=1\n", 580 | "self.name='Thread-7', i=2\n", 581 | "self.name='Thread-7', i=3\n", 582 | "self.name='Thread-7', i=4\n", 583 | "self.name='Thread-7', i=5\n", 584 | "self.name='Thread-7', i=6\n", 585 | "self.name='Thread-7', i=7\n", 586 | "self.name='Thread-7', i=8\n", 587 | "self.name='Thread-7', i=9\n", 588 | "after join False\n" 589 | ] 590 | } 591 | ], 592 | "source": [ 593 | "class DataProcessor(threading.Thread):\n", 594 | " def __init__(self, n):\n", 595 | " super().__init__()\n", 596 | " self.n = n\n", 597 | " self.result = 0\n", 598 | "\n", 599 | " def run(self):\n", 600 | " print(f\"{self.name=}, {self.n=} -- started\")\n", 601 | "\n", 602 | " for i in range(self.n):\n", 603 | " print(f\"{self.name=}, {i=}\")\n", 604 | " self.result += i\n", 605 | " time.sleep(1)\n", 606 | "\n", 607 | "\n", 608 | "data = DataProcessor(10)\n", 609 | "print(f\"{data.is_alive()=}\")\n", 610 | "\n", 611 | "data.start()\n", 612 | "data.join()\n", 613 | "print(\"after join\", data.is_alive())" 614 | ] 615 | }, 616 | { 617 | "cell_type": "code", 618 | "execution_count": null, 619 | "id": "fd44ddc2-e254-4923-815a-de07117bc9ad", 620 | "metadata": {}, 621 | "outputs": [], 622 | "source": [] 623 | }, 624 | { 625 | "cell_type": "code", 626 | "execution_count": null, 627 | "id": "5aad567c-8d14-4340-949c-73214fc4e043", 628 | "metadata": {}, 629 | "outputs": [], 630 | "source": [] 631 | }, 632 | { 633 | "cell_type": "code", 634 | "execution_count": null, 635 | "id": "d694e026-83e6-4d47-a738-e19625624ff3", 636 | "metadata": {}, 637 | "outputs": [], 638 | "source": [] 639 | }, 640 | { 641 | "cell_type": "code", 642 | "execution_count": 38, 643 | "id": "81b9e16d-4f18-47dd-a764-d9e7edf7d3d2", 644 | "metadata": {}, 645 | "outputs": [ 646 | { 647 | "name": "stdout", 648 | "output_type": "stream", 649 | "text": [ 650 | "data.is_alive()=False\n", 651 | "self.name='Thread-8', self.n=10 -- started\n", 652 | "self.name='Thread-8', i=0\n", 653 | "self.name='Thread-8', i=1\n", 654 | "self.name='Thread-8', i=2\n", 655 | "self.name='Thread-8', i=3\n", 656 | "after join True\n" 657 | ] 658 | } 659 | ], 660 | "source": [ 661 | "data = DataProcessor(10)\n", 662 | "print(f\"{data.is_alive()=}\")\n", 663 | "\n", 664 | "data.start()\n", 665 | "data.join(4)\n", 666 | "print(\"after join\", data.is_alive())" 667 | ] 668 | }, 669 | { 670 | "cell_type": "code", 671 | "execution_count": 39, 672 | "id": "5c7ab695-00be-490d-bae9-449957bfc505", 673 | "metadata": {}, 674 | "outputs": [ 675 | { 676 | "name": "stdout", 677 | "output_type": "stream", 678 | "text": [ 679 | "after join separate True\n", 680 | "self.name='Thread-8', i=4\n", 681 | "self.name='Thread-8', i=5\n", 682 | "self.name='Thread-8', i=6\n", 683 | "self.name='Thread-8', i=7\n", 684 | "self.name='Thread-8', i=8\n", 685 | "self.name='Thread-8', i=9\n" 686 | ] 687 | } 688 | ], 689 | "source": [ 690 | "print(\"after join separate\", data.is_alive())" 691 | ] 692 | }, 693 | { 694 | "cell_type": "code", 695 | "execution_count": null, 696 | "id": "87759113-50ad-4c7b-b64c-6f996412a616", 697 | "metadata": {}, 698 | "outputs": [], 699 | "source": [] 700 | }, 701 | { 702 | "cell_type": "code", 703 | "execution_count": null, 704 | "id": "38f2a67e-d616-4e92-9b1f-e94fb93d1ccb", 705 | "metadata": {}, 706 | "outputs": [], 707 | "source": [] 708 | }, 709 | { 710 | "cell_type": "code", 711 | "execution_count": 40, 712 | "id": "ae2e5594-df63-4646-9db7-58b7802c6f24", 713 | "metadata": {}, 714 | "outputs": [], 715 | "source": [ 716 | "N = 10 ** 8\n", 717 | "\n", 718 | "def counter(a, b):\n", 719 | " while a < b:\n", 720 | " a += 1" 721 | ] 722 | }, 723 | { 724 | "cell_type": "code", 725 | "execution_count": 41, 726 | "id": "7540a713-575b-4974-8f9a-2aa5edd7b94c", 727 | "metadata": {}, 728 | "outputs": [ 729 | { 730 | "name": "stdout", 731 | "output_type": "stream", 732 | "text": [ 733 | "CPU times: user 4.59 s, sys: 101 ms, total: 4.69 s\n", 734 | "Wall time: 5.75 s\n" 735 | ] 736 | } 737 | ], 738 | "source": [ 739 | "%%time\n", 740 | "\n", 741 | "counter(0, N)" 742 | ] 743 | }, 744 | { 745 | "cell_type": "code", 746 | "execution_count": null, 747 | "id": "a8814aaf-ac96-43b7-8598-ebd16bb6e381", 748 | "metadata": {}, 749 | "outputs": [], 750 | "source": [] 751 | }, 752 | { 753 | "cell_type": "code", 754 | "execution_count": 42, 755 | "id": "a0ac2f23-3a32-47d9-91b4-8535a3a58b9b", 756 | "metadata": {}, 757 | "outputs": [ 758 | { 759 | "name": "stdout", 760 | "output_type": "stream", 761 | "text": [ 762 | "CPU times: user 4.93 s, sys: 95.2 ms, total: 5.03 s\n", 763 | "Wall time: 5.95 s\n" 764 | ] 765 | } 766 | ], 767 | "source": [ 768 | "%%time\n", 769 | "\n", 770 | "th = threading.Thread(target=counter, args=(0, N), name=\"counter_solo\")\n", 771 | "th.start()\n", 772 | "th.join()" 773 | ] 774 | }, 775 | { 776 | "cell_type": "code", 777 | "execution_count": 56, 778 | "id": "65ffd8e4-d2f4-46eb-bc6b-356ab0cfb7e3", 779 | "metadata": {}, 780 | "outputs": [ 781 | { 782 | "name": "stdout", 783 | "output_type": "stream", 784 | "text": [ 785 | "threading.current_thread().name='counter_0' -- started\n", 786 | "threading.current_thread().name='counter_1' -- started\n", 787 | "threading.current_thread().name='counter_2' -- started\n", 788 | "threading.current_thread().name='counter_3' -- started\n", 789 | "threading.current_thread().name='counter_4' -- started\n", 790 | "threading.current_thread().name='counter_5' -- started\n", 791 | "threading.current_thread().name='counter_6' -- started\n", 792 | "threading.current_thread().name='counter_7' -- started\n", 793 | "threading.current_thread().name='counter_8' -- started\n", 794 | "threading.current_thread().name='counter_9' -- started\n", 795 | "threading.current_thread().name='counter_5' -- finish\n", 796 | "threading.current_thread().name='counter_8' -- finish\n", 797 | "threading.current_thread().name='counter_4' -- finish\n", 798 | "threading.current_thread().name='counter_7' -- finish\n", 799 | "threading.current_thread().name='counter_9' -- finish\n", 800 | "threading.current_thread().name='counter_2' -- finish\n", 801 | "threading.current_thread().name='counter_6' -- finish\n", 802 | "threading.current_thread().name='counter_1' -- finish\n", 803 | "threading.current_thread().name='counter_3' -- finish\n", 804 | "threading.current_thread().name='counter_0' -- finish\n", 805 | "CPU times: user 6.03 s, sys: 352 ms, total: 6.38 s\n", 806 | "Wall time: 8.64 s\n" 807 | ] 808 | } 809 | ], 810 | "source": [ 811 | "%%time\n", 812 | "\n", 813 | "N = 10 ** 8\n", 814 | "N_THREADS = 10\n", 815 | "N_JOBS = N // N_THREADS\n", 816 | "\n", 817 | "\n", 818 | "def counter(a, b):\n", 819 | " print(f\"{threading.current_thread().name=} -- started\")\n", 820 | " while a < b:\n", 821 | " a += 1\n", 822 | " print(f\"{threading.current_thread().name=} -- finish\")\n", 823 | "\n", 824 | "\n", 825 | "threads = [\n", 826 | " threading.Thread(\n", 827 | " target=counter,\n", 828 | " args=(i * N_JOBS, (i + 1) * N_JOBS),\n", 829 | " name=f\"counter_{i}\"\n", 830 | " )\n", 831 | " for i in range(N_THREADS)\n", 832 | "]\n", 833 | "\n", 834 | "for th in threads:\n", 835 | " th.start()\n", 836 | "\n", 837 | "for th in threads:\n", 838 | " th.join()" 839 | ] 840 | }, 841 | { 842 | "cell_type": "code", 843 | "execution_count": 53, 844 | "id": "93240c57-4e18-46d1-a330-0a09ca255ff4", 845 | "metadata": {}, 846 | "outputs": [ 847 | { 848 | "name": "stdout", 849 | "output_type": "stream", 850 | "text": [ 851 | "123\n" 852 | ] 853 | } 854 | ], 855 | "source": [ 856 | "print(123)" 857 | ] 858 | }, 859 | { 860 | "cell_type": "code", 861 | "execution_count": null, 862 | "id": "4db9197e-d1d9-48f0-a4ac-3c9153de7a3c", 863 | "metadata": {}, 864 | "outputs": [], 865 | "source": [] 866 | }, 867 | { 868 | "cell_type": "code", 869 | "execution_count": null, 870 | "id": "357d0bdd-a7ee-4eda-8d31-a0d818ee5576", 871 | "metadata": {}, 872 | "outputs": [], 873 | "source": [] 874 | }, 875 | { 876 | "cell_type": "code", 877 | "execution_count": 60, 878 | "id": "7d5dfef0-d142-4aab-8cca-a2022a631b13", 879 | "metadata": {}, 880 | "outputs": [ 881 | { 882 | "name": "stdout", 883 | "output_type": "stream", 884 | "text": [ 885 | "START count=0\n", 886 | "FINISH count=3656761\n", 887 | "CPU times: user 4.19 s, sys: 179 ms, total: 4.37 s\n", 888 | "Wall time: 5.27 s\n" 889 | ] 890 | } 891 | ], 892 | "source": [ 893 | "%%time\n", 894 | "\n", 895 | "N = 2 * 10 ** 7\n", 896 | "N_THREADS = 10\n", 897 | "N_JOBS = N // N_THREADS\n", 898 | "\n", 899 | "count = 0\n", 900 | "\n", 901 | "\n", 902 | "def counter(a, b):\n", 903 | " global count\n", 904 | " while a < b:\n", 905 | " a += 1\n", 906 | " tmp = count # 100, 100, 100 ....\n", 907 | " y = 10\n", 908 | " x = y + tmp\n", 909 | " z = min(tmp, 100)\n", 910 | " tmp += 1 # 101, 101, 101 ....\n", 911 | " count = tmp # 101, 101, 101 ....\n", 912 | "\n", 913 | "\n", 914 | "threads = [\n", 915 | " threading.Thread(\n", 916 | " target=counter,\n", 917 | " args=(i * N_JOBS, (i + 1) * N_JOBS),\n", 918 | " name=f\"counter_{i}\"\n", 919 | " )\n", 920 | " for i in range(N_THREADS)\n", 921 | "]\n", 922 | "\n", 923 | "print(f\"START {count=}\")\n", 924 | "\n", 925 | "for th in threads:\n", 926 | " th.start()\n", 927 | "\n", 928 | "for th in threads:\n", 929 | " th.join()\n", 930 | "\n", 931 | "print(f\"FINISH {count=}\")" 932 | ] 933 | }, 934 | { 935 | "cell_type": "code", 936 | "execution_count": null, 937 | "id": "75ac81f2-9ea5-42da-9ebd-b23fcad9a06d", 938 | "metadata": {}, 939 | "outputs": [], 940 | "source": [] 941 | }, 942 | { 943 | "cell_type": "code", 944 | "execution_count": 62, 945 | "id": "1c94c602-63be-4145-bf04-e20f058341bf", 946 | "metadata": {}, 947 | "outputs": [ 948 | { 949 | "name": "stdout", 950 | "output_type": "stream", 951 | "text": [ 952 | "START count=0\n", 953 | "FINISH count=20000000\n", 954 | "CPU times: user 9.33 s, sys: 733 ms, total: 10.1 s\n", 955 | "Wall time: 11.8 s\n" 956 | ] 957 | } 958 | ], 959 | "source": [ 960 | "%%time\n", 961 | "\n", 962 | "N = 2 * 10 ** 7\n", 963 | "N_THREADS = 10\n", 964 | "N_JOBS = N // N_THREADS\n", 965 | "\n", 966 | "count = 0\n", 967 | "lock = threading.Lock()\n", 968 | "\n", 969 | "\n", 970 | "def counter(a, b):\n", 971 | " global count\n", 972 | " while a < b:\n", 973 | " a += 1\n", 974 | " \n", 975 | " with lock: # 1\n", 976 | " tmp = count # 1 GIL aq\n", 977 | " y = 10 # 1 GIL rel\n", 978 | " x = y + tmp # 1 GIL aq\n", 979 | " z = min(tmp, 100) # 2 GIL aq\n", 980 | " tmp += 1\n", 981 | " count = tmp # 2 GIL rel\n", 982 | "\n", 983 | "\n", 984 | "threads = [\n", 985 | " threading.Thread(\n", 986 | " target=counter,\n", 987 | " args=(i * N_JOBS, (i + 1) * N_JOBS),\n", 988 | " name=f\"counter_{i}\"\n", 989 | " )\n", 990 | " for i in range(N_THREADS)\n", 991 | "]\n", 992 | "\n", 993 | "print(f\"START {count=}\")\n", 994 | "\n", 995 | "for th in threads:\n", 996 | " th.start()\n", 997 | "\n", 998 | "for th in threads:\n", 999 | " th.join()\n", 1000 | "\n", 1001 | "print(f\"FINISH {count=}\")" 1002 | ] 1003 | }, 1004 | { 1005 | "cell_type": "code", 1006 | "execution_count": 63, 1007 | "id": "8f3875cf-5514-4800-b979-3f4b75404207", 1008 | "metadata": {}, 1009 | "outputs": [ 1010 | { 1011 | "data": { 1012 | "text/plain": [ 1013 | "[<_MainThread(MainThread, started 140704311966528)>,\n", 1014 | " ,\n", 1015 | " ,\n", 1016 | " ,\n", 1017 | " ,\n", 1018 | " ,\n", 1019 | " ,\n", 1020 | " ]" 1021 | ] 1022 | }, 1023 | "execution_count": 63, 1024 | "metadata": {}, 1025 | "output_type": "execute_result" 1026 | } 1027 | ], 1028 | "source": [ 1029 | "threading.enumerate()" 1030 | ] 1031 | }, 1032 | { 1033 | "cell_type": "code", 1034 | "execution_count": null, 1035 | "id": "68fbb5ae-b57a-4c28-81bc-5b164e3b7863", 1036 | "metadata": {}, 1037 | "outputs": [], 1038 | "source": [] 1039 | }, 1040 | { 1041 | "cell_type": "code", 1042 | "execution_count": null, 1043 | "id": "8193957c-0d03-43e3-b247-43cdfc8946da", 1044 | "metadata": {}, 1045 | "outputs": [], 1046 | "source": [] 1047 | }, 1048 | { 1049 | "cell_type": "code", 1050 | "execution_count": null, 1051 | "id": "cf7f0d39-6297-4715-ad5c-351fb085170d", 1052 | "metadata": {}, 1053 | "outputs": [], 1054 | "source": [] 1055 | }, 1056 | { 1057 | "cell_type": "code", 1058 | "execution_count": 26, 1059 | "id": "d85dde08-2723-4f0f-83d4-563ca1c34ba3", 1060 | "metadata": {}, 1061 | "outputs": [], 1062 | "source": [ 1063 | "from urllib.request import urlopen\n", 1064 | "import time\n", 1065 | "import threading" 1066 | ] 1067 | }, 1068 | { 1069 | "cell_type": "code", 1070 | "execution_count": 2, 1071 | "id": "a01510f9-d51d-4b4e-b37c-96e446251a9e", 1072 | "metadata": {}, 1073 | "outputs": [], 1074 | "source": [ 1075 | "URL = \"http://ru.wikipedia.org/wiki/Python\"" 1076 | ] 1077 | }, 1078 | { 1079 | "cell_type": "code", 1080 | "execution_count": 3, 1081 | "id": "9239323c-b7b8-4d93-be28-5facaa110d6c", 1082 | "metadata": {}, 1083 | "outputs": [], 1084 | "source": [ 1085 | "def fetch_url(url):\n", 1086 | " return urlopen(url)" 1087 | ] 1088 | }, 1089 | { 1090 | "cell_type": "code", 1091 | "execution_count": 4, 1092 | "id": "49b143e4-131f-45da-94da-794dc1773825", 1093 | "metadata": {}, 1094 | "outputs": [], 1095 | "source": [ 1096 | "resp = fetch_url(URL)" 1097 | ] 1098 | }, 1099 | { 1100 | "cell_type": "code", 1101 | "execution_count": 10, 1102 | "id": "aea4e762-5046-4b3c-a91f-ff203809a3e4", 1103 | "metadata": {}, 1104 | "outputs": [ 1105 | { 1106 | "data": { 1107 | "text/plain": [ 1108 | "200" 1109 | ] 1110 | }, 1111 | "execution_count": 10, 1112 | "metadata": {}, 1113 | "output_type": "execute_result" 1114 | } 1115 | ], 1116 | "source": [ 1117 | "resp.code" 1118 | ] 1119 | }, 1120 | { 1121 | "cell_type": "code", 1122 | "execution_count": 12, 1123 | "id": "2d7f3ceb-7aff-44b6-9c0d-dd1a8e0ae1ac", 1124 | "metadata": {}, 1125 | "outputs": [ 1126 | { 1127 | "name": "stdout", 1128 | "output_type": "stream", 1129 | "text": [ 1130 | "CPU times: user 88.1 ms, sys: 31.5 ms, total: 120 ms\n", 1131 | "Wall time: 12.9 s\n" 1132 | ] 1133 | } 1134 | ], 1135 | "source": [ 1136 | "%%time\n", 1137 | "\n", 1138 | "URL = \"http://ru.wikipedia.org/wiki/Python\"\n", 1139 | "URLS = [URL] * 20\n", 1140 | "\n", 1141 | "\n", 1142 | "def fetch_url(url):\n", 1143 | " return urlopen(url)\n", 1144 | "\n", 1145 | "\n", 1146 | "def fetch_batch_urls(urls):\n", 1147 | " for url in urls:\n", 1148 | " fetch_url(url)\n", 1149 | "\n", 1150 | "\n", 1151 | "fetch_batch_urls(URLS)" 1152 | ] 1153 | }, 1154 | { 1155 | "cell_type": "code", 1156 | "execution_count": null, 1157 | "id": "cfdfb1d8-4ce2-45da-aeaf-17de922096cc", 1158 | "metadata": {}, 1159 | "outputs": [], 1160 | "source": [] 1161 | }, 1162 | { 1163 | "cell_type": "code", 1164 | "execution_count": 21, 1165 | "id": "e633bae7-d429-4778-b839-9ca127e0c2aa", 1166 | "metadata": {}, 1167 | "outputs": [ 1168 | { 1169 | "name": "stdout", 1170 | "output_type": "stream", 1171 | "text": [ 1172 | "threading.current_thread().name='fetch_batch_urls_0' -- started 2\n", 1173 | "threading.current_thread().name='fetch_batch_urls_1' -- started 2\n", 1174 | "threading.current_thread().name='fetch_batch_urls_2' -- started 2\n", 1175 | "threading.current_thread().name='fetch_batch_urls_3' -- started 2\n", 1176 | "threading.current_thread().name='fetch_batch_urls_4' -- started 2\n", 1177 | "threading.current_thread().name='fetch_batch_urls_5' -- started 2\n", 1178 | "threading.current_thread().name='fetch_batch_urls_6' -- started 2\n", 1179 | "threading.current_thread().name='fetch_batch_urls_7' -- started 2\n", 1180 | "threading.current_thread().name='fetch_batch_urls_8' -- started 2\n", 1181 | "threading.current_thread().name='fetch_batch_urls_9' -- started 2\n", 1182 | "threading.current_thread().name='fetch_batch_urls_0' -- finishthreading.current_thread().name='fetch_batch_urls_1' -- finish 2\n", 1183 | " 2\n", 1184 | "threading.current_thread().name='fetch_batch_urls_6' -- finish 2\n", 1185 | "threading.current_thread().name='fetch_batch_urls_5' -- finish 2\n", 1186 | "threading.current_thread().name='fetch_batch_urls_4' -- finishthreading.current_thread().name='fetch_batch_urls_7' -- finish 2\n", 1187 | "threading.current_thread().name='fetch_batch_urls_9' -- finish 2\n", 1188 | "threading.current_thread().name='fetch_batch_urls_3' -- finish 2\n", 1189 | " 2\n", 1190 | "threading.current_thread().name='fetch_batch_urls_8' -- finish 2\n", 1191 | "threading.current_thread().name='fetch_batch_urls_2' -- finish 2\n", 1192 | "CPU times: user 101 ms, sys: 36.2 ms, total: 137 ms\n", 1193 | "Wall time: 8.6 s\n" 1194 | ] 1195 | } 1196 | ], 1197 | "source": [ 1198 | "%%time\n", 1199 | "\n", 1200 | "URL = \"http://ru.wikipedia.org/wiki/Python\"\n", 1201 | "URLS = [URL] * 20\n", 1202 | "N_THREADS = 10\n", 1203 | "N_JOBS = len(URLS) // N_THREADS\n", 1204 | "\n", 1205 | "\n", 1206 | "def fetch_url(url):\n", 1207 | " return urlopen(url)\n", 1208 | "\n", 1209 | "\n", 1210 | "def fetch_batch_urls(urls):\n", 1211 | " print(f\"{threading.current_thread().name=} -- started\", len(urls))\n", 1212 | " for url in urls:\n", 1213 | " fetch_url(url)\n", 1214 | " print(f\"{threading.current_thread().name=} -- finish\", len(urls))\n", 1215 | "\n", 1216 | "\n", 1217 | "def run():\n", 1218 | " threads = [\n", 1219 | " threading.Thread(\n", 1220 | " target=fetch_batch_urls,\n", 1221 | " args=(URLS[i * N_JOBS : (i + 1) * N_JOBS],),\n", 1222 | " name=f\"fetch_batch_urls_{i}\"\n", 1223 | " )\n", 1224 | " for i in range(N_THREADS)\n", 1225 | " ]\n", 1226 | " \n", 1227 | " for th in threads:\n", 1228 | " th.start()\n", 1229 | " \n", 1230 | " for th in threads:\n", 1231 | " th.join()\n", 1232 | "\n", 1233 | "\n", 1234 | "run()" 1235 | ] 1236 | }, 1237 | { 1238 | "cell_type": "code", 1239 | "execution_count": 24, 1240 | "id": "47ee8b25-5585-4cf2-bf4d-6965e426c3c0", 1241 | "metadata": {}, 1242 | "outputs": [ 1243 | { 1244 | "name": "stdout", 1245 | "output_type": "stream", 1246 | "text": [ 1247 | "CPU times: user 118 ms, sys: 2.04 s, total: 2.16 s\n", 1248 | "Wall time: 2.46 s\n" 1249 | ] 1250 | } 1251 | ], 1252 | "source": [ 1253 | "%%time\n", 1254 | "\n", 1255 | "URL = \"http://ru.wikipedia.org/wiki/Python\"\n", 1256 | "URLS = [URL] * 2000\n", 1257 | "N_THREADS = 1000\n", 1258 | "N_JOBS = len(URLS) // N_THREADS\n", 1259 | "\n", 1260 | "\n", 1261 | "def fetch_url(url):\n", 1262 | " time.sleep(1)\n", 1263 | "\n", 1264 | "\n", 1265 | "def fetch_batch_urls(urls):\n", 1266 | " for url in urls:\n", 1267 | " fetch_url(url)\n", 1268 | "\n", 1269 | "def run():\n", 1270 | " threads = [\n", 1271 | " threading.Thread(\n", 1272 | " target=fetch_batch_urls,\n", 1273 | " args=(URLS[i * N_JOBS : (i + 1) * N_JOBS],),\n", 1274 | " name=f\"fetch_batch_urls_{i}\"\n", 1275 | " )\n", 1276 | " for i in range(N_THREADS)\n", 1277 | " ]\n", 1278 | " \n", 1279 | " for th in threads:\n", 1280 | " th.start()\n", 1281 | " \n", 1282 | " for th in threads:\n", 1283 | " th.join()\n", 1284 | "\n", 1285 | "\n", 1286 | "run()" 1287 | ] 1288 | }, 1289 | { 1290 | "cell_type": "code", 1291 | "execution_count": 29, 1292 | "id": "1c060d4a-23f7-4564-9384-29793ad11fe3", 1293 | "metadata": {}, 1294 | "outputs": [ 1295 | { 1296 | "name": "stdout", 1297 | "output_type": "stream", 1298 | "text": [ 1299 | "threading.current_thread().name='fetch_batch_urls_0' -- started 2\n", 1300 | "threading.current_thread().name='fetch_batch_urls_1' -- started 2\n", 1301 | "threading.current_thread().name='fetch_batch_urls_2' -- started 2\n", 1302 | "threading.current_thread().name='fetch_batch_urls_3' -- started 2\n", 1303 | "threading.current_thread().name='fetch_batch_urls_4' -- started 2\n", 1304 | "threading.current_thread().name='fetch_batch_urls_5' -- started 2\n", 1305 | "threading.current_thread().name='fetch_batch_urls_6' -- started 2\n", 1306 | "threading.current_thread().name='fetch_batch_urls_7' -- started 2\n", 1307 | "threading.current_thread().name='fetch_batch_urls_8' -- started 2\n", 1308 | "threading.current_thread().name='fetch_batch_urls_9' -- started 2\n", 1309 | "threading.current_thread().name='fetch_batch_urls_2' -- finish 6.4928669929504395\n", 1310 | "threading.current_thread().name='fetch_batch_urls_9' -- finish 6.516428232192993\n", 1311 | "threading.current_thread().name='fetch_batch_urls_7' -- finish 7.315694808959961\n", 1312 | "threading.current_thread().name='fetch_batch_urls_6' -- finish 7.316965818405151\n", 1313 | "threading.current_thread().name='fetch_batch_urls_8' -- finish 7.320839166641235\n", 1314 | "threading.current_thread().name='fetch_batch_urls_1' -- finish 7.3603692054748535\n", 1315 | "threading.current_thread().name='fetch_batch_urls_5' -- finish 7.357841730117798\n", 1316 | "threading.current_thread().name='fetch_batch_urls_4' -- finish 7.358780860900879\n", 1317 | "threading.current_thread().name='fetch_batch_urls_3' -- finish 7.4433510303497314\n", 1318 | "threading.current_thread().name='fetch_batch_urls_0' -- finish 7.538039207458496\n", 1319 | "CPU times: user 92.9 ms, sys: 32.6 ms, total: 126 ms\n", 1320 | "Wall time: 7.54 s\n" 1321 | ] 1322 | } 1323 | ], 1324 | "source": [ 1325 | "%%time\n", 1326 | "\n", 1327 | "URL = \"http://ru.wikipedia.org/wiki/Python\"\n", 1328 | "URLS = [URL] * 20\n", 1329 | "N_THREADS = 10\n", 1330 | "N_JOBS = len(URLS) // N_THREADS\n", 1331 | "\n", 1332 | "\n", 1333 | "def fetch_url(url, sem):\n", 1334 | " with sem:\n", 1335 | " return urlopen(url)\n", 1336 | "\n", 1337 | "\n", 1338 | "def fetch_batch_urls(urls, sem):\n", 1339 | " print(f\"{threading.current_thread().name=} -- started\", len(urls))\n", 1340 | " t1 = time.time()\n", 1341 | " for url in urls:\n", 1342 | " fetch_url(url, sem)\n", 1343 | " print(f\"{threading.current_thread().name=} -- finish\", time.time() - t1)\n", 1344 | "\n", 1345 | "\n", 1346 | "def run():\n", 1347 | " sem = threading.Semaphore(10)\n", 1348 | "\n", 1349 | " threads = [\n", 1350 | " threading.Thread(\n", 1351 | " target=fetch_batch_urls,\n", 1352 | " args=(URLS[i * N_JOBS : (i + 1) * N_JOBS], sem),\n", 1353 | " name=f\"fetch_batch_urls_{i}\"\n", 1354 | " )\n", 1355 | " for i in range(N_THREADS)\n", 1356 | " ]\n", 1357 | " \n", 1358 | " for th in threads:\n", 1359 | " th.start()\n", 1360 | " \n", 1361 | " for th in threads:\n", 1362 | " th.join()\n", 1363 | "\n", 1364 | "\n", 1365 | "run()" 1366 | ] 1367 | }, 1368 | { 1369 | "cell_type": "code", 1370 | "execution_count": null, 1371 | "id": "bb7224fa-9a40-45aa-8c62-28d221252ba6", 1372 | "metadata": {}, 1373 | "outputs": [], 1374 | "source": [] 1375 | }, 1376 | { 1377 | "cell_type": "code", 1378 | "execution_count": 30, 1379 | "id": "a35b45fa-8f28-4b8e-a9de-79613b78457f", 1380 | "metadata": {}, 1381 | "outputs": [], 1382 | "source": [ 1383 | "import queue" 1384 | ] 1385 | }, 1386 | { 1387 | "cell_type": "code", 1388 | "execution_count": 35, 1389 | "id": "23fe49f5-b0fb-435f-adb3-4e67f6474189", 1390 | "metadata": {}, 1391 | "outputs": [ 1392 | { 1393 | "name": "stdout", 1394 | "output_type": "stream", 1395 | "text": [ 1396 | "threading.current_thread().name='worker_fetch_0' -- started\n", 1397 | "threading.current_thread().name='worker_fetch_1' -- started\n", 1398 | "threading.current_thread().name='worker_fetch_2' -- started\n", 1399 | "threading.current_thread().name='worker_fetch_3' -- started\n", 1400 | "threading.current_thread().name='worker_fetch_4' -- started\n", 1401 | "threading.current_thread().name='worker_fetch_3' -- finished 6\n", 1402 | "threading.current_thread().name='worker_fetch_4' -- finished 6\n", 1403 | "threading.current_thread().name='worker_fetch_2' -- finished 6\n", 1404 | "threading.current_thread().name='worker_fetch_0' -- finished 6\n", 1405 | "threading.current_thread().name='worker_fetch_1' -- finished 6\n", 1406 | "[200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200]\n", 1407 | "CPU times: user 170 ms, sys: 61.2 ms, total: 231 ms\n", 1408 | "Wall time: 10 s\n" 1409 | ] 1410 | } 1411 | ], 1412 | "source": [ 1413 | "%%time\n", 1414 | "\n", 1415 | "URL = \"http://ru.wikipedia.org/wiki/Python\"\n", 1416 | "URLS = [URL] * 30\n", 1417 | "N_THREADS = 5\n", 1418 | "\n", 1419 | "\n", 1420 | "def fetch_url(url, sem, results):\n", 1421 | " with sem:\n", 1422 | " resp = urlopen(url)\n", 1423 | " results.append(resp.code)\n", 1424 | "\n", 1425 | "\n", 1426 | "def worker_fetch(que, sem, results):\n", 1427 | " print(f\"{threading.current_thread().name=} -- started\")\n", 1428 | " count = 0\n", 1429 | "\n", 1430 | " while True:\n", 1431 | " url = que.get()\n", 1432 | " if url is None:\n", 1433 | " que.put(None)\n", 1434 | " break\n", 1435 | "\n", 1436 | " fetch_url(url, sem, results)\n", 1437 | " count += 1\n", 1438 | "\n", 1439 | " print(f\"{threading.current_thread().name=} -- finished\", count)\n", 1440 | "\n", 1441 | "\n", 1442 | "def run():\n", 1443 | " sem = threading.Semaphore(10)\n", 1444 | " que = queue.Queue()\n", 1445 | " results = []\n", 1446 | "\n", 1447 | " for url in URLS:\n", 1448 | " que.put(url)\n", 1449 | "\n", 1450 | " que.put(None)\n", 1451 | "\n", 1452 | " threads = [\n", 1453 | " threading.Thread(\n", 1454 | " target=worker_fetch,\n", 1455 | " args=(que, sem, results),\n", 1456 | " name=f\"worker_fetch_{i}\"\n", 1457 | " )\n", 1458 | " for i in range(N_THREADS)\n", 1459 | " ]\n", 1460 | " \n", 1461 | " for th in threads:\n", 1462 | " th.start()\n", 1463 | " \n", 1464 | " for th in threads:\n", 1465 | " th.join()\n", 1466 | "\n", 1467 | " print(results)\n", 1468 | "\n", 1469 | "\n", 1470 | "run()" 1471 | ] 1472 | }, 1473 | { 1474 | "cell_type": "code", 1475 | "execution_count": 36, 1476 | "id": "ddda4e40-3383-4cba-a0fb-54c299544aa0", 1477 | "metadata": {}, 1478 | "outputs": [], 1479 | "source": [ 1480 | "q = queue.Queue()" 1481 | ] 1482 | }, 1483 | { 1484 | "cell_type": "code", 1485 | "execution_count": 38, 1486 | "id": "4d2317f9-7758-4e12-b26c-b260ffed19c0", 1487 | "metadata": {}, 1488 | "outputs": [ 1489 | { 1490 | "ename": "Empty", 1491 | "evalue": "", 1492 | "output_type": "error", 1493 | "traceback": [ 1494 | "\u001b[31m---------------------------------------------------------------------------\u001b[39m", 1495 | "\u001b[31mEmpty\u001b[39m Traceback (most recent call last)", 1496 | "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[38]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m \u001b[43mq\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[32;43m2\u001b[39;49m\u001b[43m)\u001b[49m\n", 1497 | "\u001b[36mFile \u001b[39m\u001b[32m/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/queue.py:212\u001b[39m, in \u001b[36mQueue.get\u001b[39m\u001b[34m(self, block, timeout)\u001b[39m\n\u001b[32m 210\u001b[39m remaining = endtime - time()\n\u001b[32m 211\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m remaining <= \u001b[32m0.0\u001b[39m:\n\u001b[32m--> \u001b[39m\u001b[32m212\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m Empty\n\u001b[32m 213\u001b[39m \u001b[38;5;28mself\u001b[39m.not_empty.wait(remaining)\n\u001b[32m 214\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.is_shutdown \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m._qsize():\n", 1498 | "\u001b[31mEmpty\u001b[39m: " 1499 | ] 1500 | } 1501 | ], 1502 | "source": [ 1503 | "q.get(timeout=2)" 1504 | ] 1505 | }, 1506 | { 1507 | "cell_type": "code", 1508 | "execution_count": null, 1509 | "id": "20d0c6d8-300d-4140-863c-56420f312e8d", 1510 | "metadata": {}, 1511 | "outputs": [], 1512 | "source": [] 1513 | } 1514 | ], 1515 | "metadata": { 1516 | "kernelspec": { 1517 | "display_name": "Python 3 (ipykernel)", 1518 | "language": "python", 1519 | "name": "python3" 1520 | }, 1521 | "language_info": { 1522 | "codemirror_mode": { 1523 | "name": "ipython", 1524 | "version": 3 1525 | }, 1526 | "file_extension": ".py", 1527 | "mimetype": "text/x-python", 1528 | "name": "python", 1529 | "nbconvert_exporter": "python", 1530 | "pygments_lexer": "ipython3", 1531 | "version": "3.13.2" 1532 | } 1533 | }, 1534 | "nbformat": 4, 1535 | "nbformat_minor": 5 1536 | } 1537 | -------------------------------------------------------------------------------- /lesson-06/homework.md: -------------------------------------------------------------------------------- 1 | # Домашнее задание #06 (потоки, процессы) 2 | 3 | ### 1. Клиент-серверное приложение для обкачки набора урлов с ограничением нагрузки 4 | #### Cервер 5 | master-worker cервер для обработки запросов от клиента. 6 | 7 | Алгоритм должен быть следующим: 8 | 9 | - Сервер должен поддерживать взаимодействие с любым числом клиентов; 10 | - Мастер и воркеры это разные потоки в едином приложении сервера; 11 | - Количество воркеров задается при запуске; 12 | - Мастер слушает порт, на который клиенты будут по TCP отправлять урлы для обкачки; 13 | - Мастер принимает запроc и передает его одному из воркеров; 14 | - Воркер читает url от клиента; 15 | - Воркер обкачивает url по http и возвращает клиенту топ K самых частых слов и их частоту в формате json {"word1": 10, "word2": 5}; 16 | - После каждого обработанного урла сервер должен вывести статистику: сколько урлов было обработано на данный момент суммарно всеми воркерами; 17 | 18 | `python server.py -w 10 -k 7` (сервер использует 10 воркеров для обкачки и отправляет клиенту топ-7 частых слов) 19 | 20 | Все действия должны быть выделены в классы/функции. 21 | 22 | #### Клиент 23 | Утилита, отправляющая запросы с урлами серверу по TCP в несколько потоков. 24 | Нужно сделать следующее: 25 | 26 | - Подготовить файл с запросами (порядка 100 разных url); 27 | - На вход клиенту передаётся два аргумента --- файл с URL'ами и количество потоков M; 28 | - Клиент создает M потоков, отправляет запросы на сервер в каждом потоке и печатает ответ сервера в стандартый вывод, например: `xxx.com: {'word1': 100, 'word2': 50}`. 29 | 30 | `python client.py 10 urls.txt` 31 | 32 | 33 | Все действия должны быть выделены в классы/функции. 34 | 35 | ### 2. Тесты для клиента и для сервера 36 | 37 | ### 3. Зеленый пайплайн в репе 38 | Обязательно: тесты, покрытие, flake8, pylint. 39 | Опционально можно добавить другие инструменты, например, mypy и black. 40 | Покрытие тестов должно составлять не менее 90%. 41 | -------------------------------------------------------------------------------- /lesson-06/lesson-06.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mailcourses/deep_python_spring_2025/ede9fa206266b26d1da2c1857c8c41482677c3b9/lesson-06/lesson-06.pdf -------------------------------------------------------------------------------- /lesson-07/class_07.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 17, 6 | "id": "a947fd7e-b6cc-4bc2-98eb-29c75485c5b8", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import multiprocessing as mp\n", 11 | "import socket\n", 12 | "import time" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 2, 18 | "id": "f904e195-36aa-4434-9437-64c5857594ea", 19 | "metadata": {}, 20 | "outputs": [ 21 | { 22 | "data": { 23 | "text/plain": [ 24 | "(['spawn', 'fork', 'forkserver'], 'spawn')" 25 | ] 26 | }, 27 | "execution_count": 2, 28 | "metadata": {}, 29 | "output_type": "execute_result" 30 | } 31 | ], 32 | "source": [ 33 | "mp.get_all_start_methods(), mp.get_start_method()" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": 3, 39 | "id": "8a79ca63-cbbb-4724-b5bb-06367e337921", 40 | "metadata": {}, 41 | "outputs": [ 42 | { 43 | "name": "stderr", 44 | "output_type": "stream", 45 | "text": [ 46 | "Traceback (most recent call last):\n", 47 | " File \u001b[35m\"\"\u001b[0m, line \u001b[35m1\u001b[0m, in \u001b[35m\u001b[0m\n", 48 | " from multiprocessing.spawn import spawn_main; \u001b[31mspawn_main\u001b[0m\u001b[1;31m(tracker_fd=66, pipe_handle=82)\u001b[0m\n", 49 | " \u001b[31m~~~~~~~~~~\u001b[0m\u001b[1;31m^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", 50 | " File \u001b[35m\"/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/multiprocessing/spawn.py\"\u001b[0m, line \u001b[35m122\u001b[0m, in \u001b[35mspawn_main\u001b[0m\n", 51 | " exitcode = _main(fd, parent_sentinel)\n", 52 | " File \u001b[35m\"/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/multiprocessing/spawn.py\"\u001b[0m, line \u001b[35m132\u001b[0m, in \u001b[35m_main\u001b[0m\n", 53 | " self = reduction.pickle.load(from_parent)\n", 54 | "\u001b[1;35mAttributeError\u001b[0m: \u001b[35mCan't get attribute 'print_data' on )>\u001b[0m\n" 55 | ] 56 | } 57 | ], 58 | "source": [ 59 | "def print_data():\n", 60 | " print(\"42\")\n", 61 | "\n", 62 | "proc = mp.Process(target=print_data)\n", 63 | "proc.start()\n", 64 | "proc.join()" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 18, 70 | "id": "f53726f0-f669-45ef-ac26-5774ada90871", 71 | "metadata": {}, 72 | "outputs": [ 73 | { 74 | "name": "stdout", 75 | "output_type": "stream", 76 | "text": [ 77 | "42\n" 78 | ] 79 | } 80 | ], 81 | "source": [ 82 | "def print_data():\n", 83 | " print(\"42\")\n", 84 | "\n", 85 | "\n", 86 | "proc_ctx = mp.get_context(\"fork\")\n", 87 | "\n", 88 | "proc = proc_ctx.Process(target=print_data)\n", 89 | "proc.start()\n", 90 | "proc.join()" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": 10, 96 | "id": "de94eaee-11eb-4af5-bdb0-a00da1e58376", 97 | "metadata": {}, 98 | "outputs": [ 99 | { 100 | "data": { 101 | "text/plain": [ 102 | "'spawn'" 103 | ] 104 | }, 105 | "execution_count": 10, 106 | "metadata": {}, 107 | "output_type": "execute_result" 108 | } 109 | ], 110 | "source": [ 111 | "mp.get_start_method()" 112 | ] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "execution_count": 11, 117 | "id": "3598c9c5-12cb-415a-ba1c-94ebdd2f025e", 118 | "metadata": {}, 119 | "outputs": [], 120 | "source": [ 121 | "class Person:\n", 122 | " def __init__(self, name):\n", 123 | " self.name = name\n", 124 | "\n", 125 | "\n", 126 | "def update_data(data):\n", 127 | " print(\"update_data\", data)\n", 128 | "\n", 129 | " data[1] = \"one\"\n", 130 | " data[(\"str\", 42)] = \"str42\"\n", 131 | " data[\"pers\"] = Person(\"steve\")\n", 132 | " \n", 133 | " print(\"finished update_data\", data)" 134 | ] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "execution_count": 14, 139 | "id": "113594a1-2840-4d74-b20c-30f1abca3ed4", 140 | "metadata": {}, 141 | "outputs": [ 142 | { 143 | "name": "stdout", 144 | "output_type": "stream", 145 | "text": [ 146 | "update_data {}\n", 147 | "finished update_data {1: 'one', ('str', 42): 'str42', 'pers': <__main__.Person object at 0x112cffed0>}\n" 148 | ] 149 | } 150 | ], 151 | "source": [ 152 | "update_data({})" 153 | ] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "execution_count": 15, 158 | "id": "3e3e8f0c-9118-497c-b80d-ca2deff3a741", 159 | "metadata": {}, 160 | "outputs": [ 161 | { 162 | "name": "stdout", 163 | "output_type": "stream", 164 | "text": [ 165 | "data before update {1: 'initial_one'}\n", 166 | "update_data {1: 'initial_one'}\n", 167 | "finished update_data {1: 'one', ('str', 42): 'str42', 'pers': <__main__.Person object at 0x1131163f0>}\n", 168 | "data after update {1: 'initial_one'}\n" 169 | ] 170 | } 171 | ], 172 | "source": [ 173 | "data = {1: \"initial_one\"}\n", 174 | "\n", 175 | "print(\"data before update\", data)\n", 176 | "\n", 177 | "proc = proc_ctx.Process(target=update_data, args=(data,))\n", 178 | "proc.start()\n", 179 | "proc.join()\n", 180 | "\n", 181 | "print(\"data after update\", data)" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "id": "fb5b14ee-cef1-4a91-8299-05d1945905f4", 188 | "metadata": {}, 189 | "outputs": [], 190 | "source": [] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": null, 195 | "id": "da1055d2-c35f-4677-9870-45b921a515b0", 196 | "metadata": {}, 197 | "outputs": [], 198 | "source": [] 199 | }, 200 | { 201 | "cell_type": "code", 202 | "execution_count": 23, 203 | "id": "eefd3467-945e-4860-931c-884691d7158f", 204 | "metadata": {}, 205 | "outputs": [ 206 | { 207 | "name": "stdout", 208 | "output_type": "stream", 209 | "text": [ 210 | "data before update {1: 'initial_one'}\n", 211 | "update_data {1: 'initial_one'}\n", 212 | "finished update_data {1: 'one', ('str', 42): 'str42', 'pers': <__main__.Person object at 0x1123d6cf0>}\n", 213 | "data after update {1: 'one', ('str', 42): 'str42', 'pers': <__main__.Person object at 0x1123d6cf0>}\n" 214 | ] 215 | } 216 | ], 217 | "source": [ 218 | "class Person:\n", 219 | " def __init__(self, name):\n", 220 | " self.name = name\n", 221 | "\n", 222 | "\n", 223 | "def update_data(data):\n", 224 | " print(\"update_data\", data)\n", 225 | "\n", 226 | " data[1] = \"one\"\n", 227 | " data[(\"str\", 42)] = \"str42\"\n", 228 | " data[\"pers\"] = Person(\"steve\")\n", 229 | "\n", 230 | " print(\"finished update_data\", data)\n", 231 | "\n", 232 | "\n", 233 | "with proc_ctx.Manager() as manager:\n", 234 | " data = manager.dict()\n", 235 | " data[1] = \"initial_one\"\n", 236 | " print(\"data before update\", data)\n", 237 | " \n", 238 | " proc = proc_ctx.Process(target=update_data, args=(data,))\n", 239 | " proc.start()\n", 240 | " proc.join()\n", 241 | "\n", 242 | " print(\"data after update\", data)\n", 243 | " backup = dict(data)" 244 | ] 245 | }, 246 | { 247 | "cell_type": "code", 248 | "execution_count": 24, 249 | "id": "758b7fcf-1502-4e8a-99d7-0033efe6b8fa", 250 | "metadata": {}, 251 | "outputs": [ 252 | { 253 | "data": { 254 | "text/plain": [ 255 | "<__main__.Person at 0x1123d4ec0>" 256 | ] 257 | }, 258 | "execution_count": 24, 259 | "metadata": {}, 260 | "output_type": "execute_result" 261 | } 262 | ], 263 | "source": [ 264 | "backup[\"pers\"]" 265 | ] 266 | }, 267 | { 268 | "cell_type": "code", 269 | "execution_count": 25, 270 | "id": "ae8ffa9d-5ba4-46f4-a14d-adbbc0627962", 271 | "metadata": {}, 272 | "outputs": [ 273 | { 274 | "data": { 275 | "text/plain": [ 276 | "{1: 'one', ('str', 42): 'str42', 'pers': <__main__.Person at 0x1123d4ec0>}" 277 | ] 278 | }, 279 | "execution_count": 25, 280 | "metadata": {}, 281 | "output_type": "execute_result" 282 | } 283 | ], 284 | "source": [ 285 | "backup" 286 | ] 287 | }, 288 | { 289 | "cell_type": "code", 290 | "execution_count": null, 291 | "id": "f6fd84e8-cb3e-409b-9568-0835464d42e6", 292 | "metadata": {}, 293 | "outputs": [], 294 | "source": [] 295 | }, 296 | { 297 | "cell_type": "code", 298 | "execution_count": 24, 299 | "id": "378da8bc-f1ac-4719-a3c0-a34cfbe9ebed", 300 | "metadata": {}, 301 | "outputs": [ 302 | { 303 | "name": "stdout", 304 | "output_type": "stream", 305 | "text": [ 306 | "start server\n", 307 | "start client\n", 308 | "server accepted client ('127.0.0.1', 50275)\n", 309 | "server finished to clientclient recv from server ('127.0.0.1', 50275)КРУЖКА\n", 310 | "\n", 311 | "client finished\n", 312 | "total finish\n", 313 | "server accepted client ('127.0.0.1', 50316)\n", 314 | "server finished to client ('127.0.0.1', 50316)\n", 315 | "server accepted client ('127.0.0.1', 50350)\n", 316 | "server finished to client ('127.0.0.1', 50350)\n", 317 | "server accepted client ('127.0.0.1', 50374)\n", 318 | "server finished to client ('127.0.0.1', 50374)\n", 319 | "server accepted client ('127.0.0.1', 50412)\n", 320 | "server finished to client ('127.0.0.1', 50412)\n", 321 | "server finished\n" 322 | ] 323 | } 324 | ], 325 | "source": [ 326 | "def server():\n", 327 | " print(\"start server\")\n", 328 | " with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:\n", 329 | " sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n", 330 | "\n", 331 | " sock.bind((\"localhost\", 20_000))\n", 332 | " sock.listen(5)\n", 333 | "\n", 334 | " i = 5\n", 335 | " while i > 0:\n", 336 | " i -= 1\n", 337 | " \n", 338 | " client_sock, address = sock.accept()\n", 339 | " print(\"server accepted client\", address)\n", 340 | "\n", 341 | " data = client_sock.recv(1048)\n", 342 | " data = data.decode().upper()\n", 343 | " client_sock.sendall(data.encode())\n", 344 | " print(\"server finished to client\", address)\n", 345 | "\n", 346 | " print(\"server finished\")\n", 347 | "\n", 348 | "\n", 349 | "def client(data):\n", 350 | " print(\"start client\")\n", 351 | " with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:\n", 352 | " sock.connect((\"localhost\", 20_000))\n", 353 | "\n", 354 | " sock.sendall(data.encode())\n", 355 | "\n", 356 | " data = sock.recv(1048).decode()\n", 357 | " print(\"client recv from server\", data)\n", 358 | "\n", 359 | " print(\"client finished\")\n", 360 | "\n", 361 | "\n", 362 | "proc_server = proc_ctx.Process(target=server)\n", 363 | "proc_client = proc_ctx.Process(target=client, args=(\"кружка\",))\n", 364 | "\n", 365 | "proc_server.start()\n", 366 | "time.sleep(1)\n", 367 | "proc_client.start()\n", 368 | "\n", 369 | "# proc_server.join()\n", 370 | "proc_client.join()\n", 371 | "\n", 372 | "print(\"total finish\")" 373 | ] 374 | }, 375 | { 376 | "cell_type": "code", 377 | "execution_count": 25, 378 | "id": "c708863e-75d3-4ba0-853b-e84a51ada687", 379 | "metadata": {}, 380 | "outputs": [ 381 | { 382 | "data": { 383 | "text/plain": [ 384 | "True" 385 | ] 386 | }, 387 | "execution_count": 25, 388 | "metadata": {}, 389 | "output_type": "execute_result" 390 | } 391 | ], 392 | "source": [ 393 | "proc_server.is_alive()" 394 | ] 395 | }, 396 | { 397 | "cell_type": "code", 398 | "execution_count": 26, 399 | "id": "2d1d467b-d28c-44ad-867f-400f08aef378", 400 | "metadata": {}, 401 | "outputs": [ 402 | { 403 | "name": "stdout", 404 | "output_type": "stream", 405 | "text": [ 406 | "start client\n", 407 | "client recv from server НОУТ\n", 408 | "client finished\n" 409 | ] 410 | } 411 | ], 412 | "source": [ 413 | "proc_client = proc_ctx.Process(target=client, args=(\"ноут\",))\n", 414 | "proc_client.start()\n", 415 | "proc_client.join()" 416 | ] 417 | }, 418 | { 419 | "cell_type": "code", 420 | "execution_count": 27, 421 | "id": "8ef70df8-90ec-4980-9723-5f8b9e66a3aa", 422 | "metadata": {}, 423 | "outputs": [ 424 | { 425 | "name": "stdout", 426 | "output_type": "stream", 427 | "text": [ 428 | "start client\n", 429 | "client recv from server НОУТ\n", 430 | "client finished\n" 431 | ] 432 | } 433 | ], 434 | "source": [ 435 | "proc_client = proc_ctx.Process(target=client, args=(\"ноут\",))\n", 436 | "proc_client.start()\n", 437 | "proc_client.join()" 438 | ] 439 | }, 440 | { 441 | "cell_type": "code", 442 | "execution_count": 28, 443 | "id": "71884bf1-3fe8-4ae9-ad5c-77140f252112", 444 | "metadata": {}, 445 | "outputs": [ 446 | { 447 | "name": "stdout", 448 | "output_type": "stream", 449 | "text": [ 450 | "start client\n", 451 | "client recv from server НОУТ\n", 452 | "client finished\n" 453 | ] 454 | } 455 | ], 456 | "source": [ 457 | "proc_client = proc_ctx.Process(target=client, args=(\"ноут\",))\n", 458 | "proc_client.start()\n", 459 | "proc_client.join()" 460 | ] 461 | }, 462 | { 463 | "cell_type": "code", 464 | "execution_count": 29, 465 | "id": "c7d3c515-d72e-4771-9a9d-2879919c67a7", 466 | "metadata": {}, 467 | "outputs": [ 468 | { 469 | "name": "stdout", 470 | "output_type": "stream", 471 | "text": [ 472 | "start client\n", 473 | "client recv from server ТЕТРАДЬ\n", 474 | "client finished\n" 475 | ] 476 | } 477 | ], 478 | "source": [ 479 | "proc_client = proc_ctx.Process(target=client, args=(\"тетрадь\",))\n", 480 | "proc_client.start()\n", 481 | "proc_client.join()" 482 | ] 483 | }, 484 | { 485 | "cell_type": "code", 486 | "execution_count": 30, 487 | "id": "a576e39a-53d1-4358-b92c-14ee83352211", 488 | "metadata": {}, 489 | "outputs": [ 490 | { 491 | "name": "stdout", 492 | "output_type": "stream", 493 | "text": [ 494 | "False\n", 495 | "start client\n" 496 | ] 497 | }, 498 | { 499 | "name": "stderr", 500 | "output_type": "stream", 501 | "text": [ 502 | "Process ForkProcess-31:\n", 503 | "Traceback (most recent call last):\n", 504 | " File \"/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/multiprocessing/process.py\", line 313, in _bootstrap\n", 505 | " self.run()\n", 506 | " ~~~~~~~~^^\n", 507 | " File \"/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/multiprocessing/process.py\", line 108, in run\n", 508 | " self._target(*self._args, **self._kwargs)\n", 509 | " ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", 510 | " File \"/var/folders/c1/9wdb7dps2x332c6l1y2hrqkh0000gp/T/ipykernel_76265/2931819710.py\", line 27, in client\n", 511 | " sock.connect((\"localhost\", 20_000))\n", 512 | " ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^\n", 513 | "ConnectionRefusedError: [Errno 61] Connection refused\n" 514 | ] 515 | } 516 | ], 517 | "source": [ 518 | "print(proc_server.is_alive())\n", 519 | "\n", 520 | "proc_client = proc_ctx.Process(target=client, args=(\"ноут\",))\n", 521 | "proc_client.start()\n", 522 | "proc_client.join()" 523 | ] 524 | }, 525 | { 526 | "cell_type": "code", 527 | "execution_count": null, 528 | "id": "4dba9c52-6939-41ab-a86c-1ec1f88e0a66", 529 | "metadata": {}, 530 | "outputs": [], 531 | "source": [] 532 | }, 533 | { 534 | "cell_type": "code", 535 | "execution_count": null, 536 | "id": "be33dc53-d95e-40e7-8a91-6326ace16876", 537 | "metadata": {}, 538 | "outputs": [], 539 | "source": [] 540 | }, 541 | { 542 | "cell_type": "code", 543 | "execution_count": null, 544 | "id": "91cd570f-6953-4706-8cb8-51eeb90946b4", 545 | "metadata": {}, 546 | "outputs": [], 547 | "source": [] 548 | }, 549 | { 550 | "cell_type": "code", 551 | "execution_count": 33, 552 | "id": "9274e9ca-e253-4ebb-86d1-be384945f866", 553 | "metadata": {}, 554 | "outputs": [], 555 | "source": [ 556 | "import asyncio" 557 | ] 558 | }, 559 | { 560 | "cell_type": "code", 561 | "execution_count": 35, 562 | "id": "0cf56d22-4206-40e7-819d-506c29fb52d7", 563 | "metadata": {}, 564 | "outputs": [ 565 | { 566 | "name": "stdout", 567 | "output_type": "stream", 568 | "text": [ 569 | "started at 21:04:06\n", 570 | "hello\n", 571 | "world\n", 572 | "finished at 21:04:09\n", 573 | "total_time 3.0034570693969727\n" 574 | ] 575 | } 576 | ], 577 | "source": [ 578 | "async def say_after(delay, what):\n", 579 | " await asyncio.sleep(delay)\n", 580 | " print(what)\n", 581 | "\n", 582 | "\n", 583 | "async def main():\n", 584 | " print(f\"started at {time.strftime('%X')}\")\n", 585 | " await say_after(1, 'hello')\n", 586 | " await say_after(2, 'world')\n", 587 | " print(f\"finished at {time.strftime('%X')}\")\n", 588 | "\n", 589 | "\n", 590 | "t1 = time.time()\n", 591 | "\n", 592 | "await main()\n", 593 | "\n", 594 | "t2 = time.time()\n", 595 | "print(\"total_time\", t2 - t1)" 596 | ] 597 | }, 598 | { 599 | "cell_type": "code", 600 | "execution_count": null, 601 | "id": "9e6d1aa4-b7da-445f-9fc6-1651436e8225", 602 | "metadata": {}, 603 | "outputs": [], 604 | "source": [] 605 | }, 606 | { 607 | "cell_type": "code", 608 | "execution_count": 36, 609 | "id": "59ccde23-a62f-4712-bafd-73f059340a1c", 610 | "metadata": {}, 611 | "outputs": [ 612 | { 613 | "name": "stdout", 614 | "output_type": "stream", 615 | "text": [ 616 | "started at 21:04:47\n", 617 | "world\n", 618 | "hello\n", 619 | "finished at 21:04:50\n", 620 | "total_time 3.002131938934326\n" 621 | ] 622 | } 623 | ], 624 | "source": [ 625 | "async def say_after(delay, what):\n", 626 | " await asyncio.sleep(delay)\n", 627 | " print(what)\n", 628 | "\n", 629 | "\n", 630 | "async def main():\n", 631 | " print(f\"started at {time.strftime('%X')}\")\n", 632 | " say1 = say_after(1, 'hello')\n", 633 | " await say_after(2, 'world')\n", 634 | " await say1\n", 635 | " print(f\"finished at {time.strftime('%X')}\")\n", 636 | "\n", 637 | "\n", 638 | "t1 = time.time()\n", 639 | "\n", 640 | "await main()\n", 641 | "\n", 642 | "t2 = time.time()\n", 643 | "print(\"total_time\", t2 - t1)" 644 | ] 645 | }, 646 | { 647 | "cell_type": "code", 648 | "execution_count": null, 649 | "id": "4d83490b-d840-4953-87eb-eb0753ed0ecb", 650 | "metadata": {}, 651 | "outputs": [], 652 | "source": [] 653 | }, 654 | { 655 | "cell_type": "code", 656 | "execution_count": null, 657 | "id": "21f3ca96-2317-45f1-8653-4a1a3ed1e69f", 658 | "metadata": {}, 659 | "outputs": [], 660 | "source": [] 661 | }, 662 | { 663 | "cell_type": "code", 664 | "execution_count": 37, 665 | "id": "2b8d5845-a6bc-4aa4-aa21-41567b8a08cd", 666 | "metadata": {}, 667 | "outputs": [ 668 | { 669 | "name": "stdout", 670 | "output_type": "stream", 671 | "text": [ 672 | "started at 21:05:42\n", 673 | "hello\n", 674 | "world\n", 675 | "finished at 21:05:44\n", 676 | "total_time 2.001389980316162\n" 677 | ] 678 | } 679 | ], 680 | "source": [ 681 | "async def say_after(delay, what):\n", 682 | " await asyncio.sleep(delay)\n", 683 | " print(what)\n", 684 | "\n", 685 | "\n", 686 | "async def main():\n", 687 | " print(f\"started at {time.strftime('%X')}\")\n", 688 | " say1 = asyncio.create_task(say_after(1, 'hello'))\n", 689 | " await say_after(2, 'world')\n", 690 | " await say1\n", 691 | " print(f\"finished at {time.strftime('%X')}\")\n", 692 | "\n", 693 | "\n", 694 | "t1 = time.time()\n", 695 | "\n", 696 | "await main()\n", 697 | "\n", 698 | "t2 = time.time()\n", 699 | "print(\"total_time\", t2 - t1)" 700 | ] 701 | }, 702 | { 703 | "cell_type": "code", 704 | "execution_count": null, 705 | "id": "0ccce981-9a67-4c5d-8a3e-13903927d923", 706 | "metadata": {}, 707 | "outputs": [], 708 | "source": [] 709 | }, 710 | { 711 | "cell_type": "code", 712 | "execution_count": null, 713 | "id": "2d8051c2-3621-4e26-8018-50f6d20032d1", 714 | "metadata": {}, 715 | "outputs": [], 716 | "source": [ 717 | "async def say_after(delay, what):\n", 718 | " await asyncio.sleep(delay)\n", 719 | " print(what)\n", 720 | "\n", 721 | "\n", 722 | "async def main():\n", 723 | " print(f\"started at {time.strftime('%X')}\")\n", 724 | " say1 = asyncio.create_task(say_after(1, 'hello'))\n", 725 | " await say_after(2, 'world')\n", 726 | " await say1\n", 727 | " print(f\"finished at {time.strftime('%X')}\")\n", 728 | "\n", 729 | "\n", 730 | "t1 = time.time()\n", 731 | "\n", 732 | "await main()\n", 733 | "\n", 734 | "t2 = time.time()\n", 735 | "print(\"total_time\", t2 - t1)" 736 | ] 737 | }, 738 | { 739 | "cell_type": "code", 740 | "execution_count": null, 741 | "id": "af8f41c2-34e2-4902-8044-64e0374b93a4", 742 | "metadata": {}, 743 | "outputs": [], 744 | "source": [] 745 | }, 746 | { 747 | "cell_type": "code", 748 | "execution_count": 38, 749 | "id": "a8099f49-3a91-40e4-a550-69c8002e8e0c", 750 | "metadata": {}, 751 | "outputs": [ 752 | { 753 | "name": "stdout", 754 | "output_type": "stream", 755 | "text": [ 756 | "started at 21:08:00\n", 757 | "world\n", 758 | "hello\n", 759 | "finished at 21:08:03\n", 760 | "total_time 3.0099399089813232\n" 761 | ] 762 | } 763 | ], 764 | "source": [ 765 | "async def say_after(delay, what):\n", 766 | " time.sleep(delay)\n", 767 | " print(what)\n", 768 | "\n", 769 | "\n", 770 | "async def main():\n", 771 | " print(f\"started at {time.strftime('%X')}\")\n", 772 | " say1 = asyncio.create_task(say_after(1, 'hello'))\n", 773 | " await say_after(2, 'world')\n", 774 | " await say1\n", 775 | " print(f\"finished at {time.strftime('%X')}\")\n", 776 | "\n", 777 | "\n", 778 | "t1 = time.time()\n", 779 | "\n", 780 | "await main()\n", 781 | "\n", 782 | "t2 = time.time()\n", 783 | "print(\"total_time\", t2 - t1)" 784 | ] 785 | }, 786 | { 787 | "cell_type": "code", 788 | "execution_count": null, 789 | "id": "ca238c17-9b4c-4956-b4f0-146b4be4f2d4", 790 | "metadata": {}, 791 | "outputs": [], 792 | "source": [] 793 | }, 794 | { 795 | "cell_type": "code", 796 | "execution_count": 45, 797 | "id": "939abd5e-0028-4f2c-be01-5133ae31abab", 798 | "metadata": {}, 799 | "outputs": [ 800 | { 801 | "name": "stdout", 802 | "output_type": "stream", 803 | "text": [ 804 | "started at 21:17:17\n", 805 | "after sleep in main\n", 806 | "hello 1 1.000389814376831\n", 807 | "world 2 2.0008420944213867\n", 808 | "finished at 21:17:21\n", 809 | "total_time 4.033520936965942\n" 810 | ] 811 | } 812 | ], 813 | "source": [ 814 | "async def say_after(delay, what):\n", 815 | " t1 = time.time()\n", 816 | " await asyncio.sleep(delay)\n", 817 | " t2 = time.time()\n", 818 | " print(what, delay, t2 - t1)\n", 819 | "\n", 820 | "\n", 821 | "async def main():\n", 822 | " print(f\"started at {time.strftime('%X')}\")\n", 823 | " \n", 824 | " say1 = asyncio.create_task(say_after(1, 'hello'))\n", 825 | " time.sleep(2)\n", 826 | " print(\"after sleep in main\")\n", 827 | " say2 = asyncio.create_task(say_after(2, 'world'))\n", 828 | " \n", 829 | " await asyncio.gather(say1, say2)\n", 830 | " print(f\"finished at {time.strftime('%X')}\")\n", 831 | "\n", 832 | "\n", 833 | "t1 = time.time()\n", 834 | "\n", 835 | "await main()\n", 836 | "\n", 837 | "t2 = time.time()\n", 838 | "print(\"total_time\", t2 - t1)" 839 | ] 840 | }, 841 | { 842 | "cell_type": "code", 843 | "execution_count": null, 844 | "id": "f97d326a-1672-4b12-b044-0ca936030410", 845 | "metadata": {}, 846 | "outputs": [], 847 | "source": [] 848 | }, 849 | { 850 | "cell_type": "code", 851 | "execution_count": 49, 852 | "id": "9a8c3876-ea81-43bf-94eb-a88a2d9efe7b", 853 | "metadata": {}, 854 | "outputs": [], 855 | "source": [ 856 | "import aiohttp" 857 | ] 858 | }, 859 | { 860 | "cell_type": "code", 861 | "execution_count": 66, 862 | "id": "c4136487-2f2c-4bab-ac21-0b38e974d0a0", 863 | "metadata": {}, 864 | "outputs": [ 865 | { 866 | "name": "stdout", 867 | "output_type": "stream", 868 | "text": [ 869 | "batch time 10.082428216934204\n" 870 | ] 871 | } 872 | ], 873 | "source": [ 874 | "URL = \"https://docs.python.org/3/whatsnew/3.13.html\"\n", 875 | "URLS = [URL] * 10\n", 876 | "\n", 877 | "\n", 878 | "async def fetch_url(session, url):\n", 879 | " await asyncio.sleep(1)\n", 880 | " # async with session.get(url) as resp:\n", 881 | " # return await resp.text()\n", 882 | "\n", 883 | "\n", 884 | "async def fetch_batch_urls(urls):\n", 885 | " async with aiohttp.ClientSession() as session:\n", 886 | " tasks = [fetch_url(session, url) for url in urls]\n", 887 | "\n", 888 | " for task in tasks:\n", 889 | " await task\n", 890 | "\n", 891 | "\n", 892 | "async def run():\n", 893 | " t1 = time.time()\n", 894 | " await fetch_batch_urls(URLS)\n", 895 | " t2 = time.time()\n", 896 | " print(\"batch time\", t2 - t1)\n", 897 | "\n", 898 | "\n", 899 | "await run()" 900 | ] 901 | }, 902 | { 903 | "cell_type": "code", 904 | "execution_count": null, 905 | "id": "f9f4ad37-ec8e-490d-ad76-2f77a2e1c3a8", 906 | "metadata": {}, 907 | "outputs": [], 908 | "source": [] 909 | }, 910 | { 911 | "cell_type": "code", 912 | "execution_count": 67, 913 | "id": "2e4e6899-687b-4baa-9b09-5551ad8beb1e", 914 | "metadata": {}, 915 | "outputs": [ 916 | { 917 | "name": "stdout", 918 | "output_type": "stream", 919 | "text": [ 920 | "10\n", 921 | "batch time 1.1302556991577148\n" 922 | ] 923 | } 924 | ], 925 | "source": [ 926 | "URL = \"https://docs.python.org/3/whatsnew/3.13.html\"\n", 927 | "URLS = [URL] * 10\n", 928 | "\n", 929 | "\n", 930 | "async def fetch_url(session, url):\n", 931 | " await asyncio.sleep(1)\n", 932 | " # async with session.get(url) as resp:\n", 933 | " # return await resp.text()\n", 934 | "\n", 935 | "\n", 936 | "async def fetch_batch_urls(urls):\n", 937 | " async with aiohttp.ClientSession() as session:\n", 938 | " tasks = [fetch_url(session, url) for url in urls]\n", 939 | "\n", 940 | " res = await asyncio.gather(*tasks)\n", 941 | " print(len(res))\n", 942 | "\n", 943 | "\n", 944 | "async def run():\n", 945 | " t1 = time.time()\n", 946 | " await fetch_batch_urls(URLS)\n", 947 | " t2 = time.time()\n", 948 | " print(\"batch time\", t2 - t1)\n", 949 | "\n", 950 | "\n", 951 | "await run()" 952 | ] 953 | }, 954 | { 955 | "cell_type": "code", 956 | "execution_count": null, 957 | "id": "62f40730-46c0-45d1-bf88-cd1305a06922", 958 | "metadata": {}, 959 | "outputs": [], 960 | "source": [] 961 | }, 962 | { 963 | "cell_type": "code", 964 | "execution_count": null, 965 | "id": "7962d5bf-b17a-4f33-b441-482267f887eb", 966 | "metadata": {}, 967 | "outputs": [], 968 | "source": [] 969 | }, 970 | { 971 | "cell_type": "code", 972 | "execution_count": null, 973 | "id": "110bee4a-b40c-4e95-8e35-131548caced3", 974 | "metadata": {}, 975 | "outputs": [], 976 | "source": [] 977 | }, 978 | { 979 | "cell_type": "code", 980 | "execution_count": null, 981 | "id": "8d5cd4ad-d74b-4830-91ce-bce8101167a8", 982 | "metadata": {}, 983 | "outputs": [], 984 | "source": [] 985 | }, 986 | { 987 | "cell_type": "code", 988 | "execution_count": 70, 989 | "id": "5d2b8d03-f629-4c6b-b5de-921586799907", 990 | "metadata": {}, 991 | "outputs": [ 992 | { 993 | "name": "stdout", 994 | "output_type": "stream", 995 | "text": [ 996 | "fetch_worker started worker_0\n", 997 | "fetch_worker started worker_1\n", 998 | "fetch_worker finished worker_0\n", 999 | "fetch_worker finished worker_1\n", 1000 | "batch time 5.003886938095093\n" 1001 | ] 1002 | } 1003 | ], 1004 | "source": [ 1005 | "URL = \"https://docs.python.org/3/whatsnew/3.13.html\"\n", 1006 | "URLS = [URL] * 10\n", 1007 | "WORKERS = 2\n", 1008 | "\n", 1009 | "\n", 1010 | "async def fetch_url(session, url):\n", 1011 | " await asyncio.sleep(1)\n", 1012 | " # async with session.get(url) as resp:\n", 1013 | " # return await resp.text()\n", 1014 | "\n", 1015 | "\n", 1016 | "async def fetch_worker(session, que, name):\n", 1017 | " print(\"fetch_worker started\", name)\n", 1018 | " while True:\n", 1019 | " url = await que.get()\n", 1020 | "\n", 1021 | " if url is None:\n", 1022 | " await que.put(url)\n", 1023 | " break\n", 1024 | "\n", 1025 | " try:\n", 1026 | " result = await fetch_url(session, url)\n", 1027 | " except Exception as err:\n", 1028 | " print(\"error\", err)\n", 1029 | "\n", 1030 | " print(\"fetch_worker finished\", name)\n", 1031 | "\n", 1032 | "\n", 1033 | "async def run():\n", 1034 | " t1 = time.time()\n", 1035 | "\n", 1036 | " que = asyncio.Queue()\n", 1037 | " for url in URLS:\n", 1038 | " await que.put(url)\n", 1039 | " await que.put(None)\n", 1040 | "\n", 1041 | " async with aiohttp.ClientSession() as session:\n", 1042 | " workers = [\n", 1043 | " fetch_worker(session, que, f\"worker_{i}\")\n", 1044 | " for i in range(WORKERS)\n", 1045 | " ]\n", 1046 | " await asyncio.gather(*workers)\n", 1047 | "\n", 1048 | " t2 = time.time()\n", 1049 | " print(\"batch time\", t2 - t1)\n", 1050 | "\n", 1051 | "\n", 1052 | "await run()" 1053 | ] 1054 | }, 1055 | { 1056 | "cell_type": "code", 1057 | "execution_count": 1, 1058 | "id": "cb1ea6f8-29bc-408d-bfe6-9593d712d410", 1059 | "metadata": {}, 1060 | "outputs": [], 1061 | "source": [ 1062 | "import asyncio\n", 1063 | "import aiohttp\n", 1064 | "import time" 1065 | ] 1066 | }, 1067 | { 1068 | "cell_type": "code", 1069 | "execution_count": 3, 1070 | "id": "ed0527d7-e104-4771-9139-b0e19710472e", 1071 | "metadata": {}, 1072 | "outputs": [ 1073 | { 1074 | "name": "stdout", 1075 | "output_type": "stream", 1076 | "text": [ 1077 | "queue\n", 1078 | "workers\n", 1079 | "fetch_worker started worker_0\n", 1080 | "fetch_worker started worker_1\n", 1081 | "batch time 5.025568008422852\n" 1082 | ] 1083 | } 1084 | ], 1085 | "source": [ 1086 | "URL = \"https://docs.python.org/3/whatsnew/3.13.html\"\n", 1087 | "URLS = [URL] * 10\n", 1088 | "WORKERS = 2\n", 1089 | "\n", 1090 | "\n", 1091 | "async def fetch_url(session, url):\n", 1092 | " await asyncio.sleep(1)\n", 1093 | " # async with session.get(url) as resp:\n", 1094 | " # return await resp.text()\n", 1095 | "\n", 1096 | "\n", 1097 | "async def fetch_worker(session, que, name):\n", 1098 | " print(\"fetch_worker started\", name)\n", 1099 | " while True:\n", 1100 | " url = await que.get()\n", 1101 | "\n", 1102 | " try:\n", 1103 | " result = await fetch_url(session, url)\n", 1104 | " except Exception as err:\n", 1105 | " print(\"error\", err)\n", 1106 | " finally:\n", 1107 | " que.task_done()\n", 1108 | "\n", 1109 | " print(\"fetch_worker finished\", name)\n", 1110 | "\n", 1111 | "\n", 1112 | "async def run():\n", 1113 | " t1 = time.time()\n", 1114 | "\n", 1115 | " que = asyncio.Queue()\n", 1116 | " for url in URLS:\n", 1117 | " await que.put(url)\n", 1118 | "\n", 1119 | " print(\"queue\")\n", 1120 | "\n", 1121 | " async with aiohttp.ClientSession() as session:\n", 1122 | " workers = [\n", 1123 | " asyncio.create_task(\n", 1124 | " fetch_worker(session, que, f\"worker_{i}\")\n", 1125 | " )\n", 1126 | " for i in range(WORKERS)\n", 1127 | " ]\n", 1128 | " print(\"workers\")\n", 1129 | "\n", 1130 | " await que.join()\n", 1131 | "\n", 1132 | " for worker in workers:\n", 1133 | " worker.cancel()\n", 1134 | "\n", 1135 | " t2 = time.time()\n", 1136 | " print(\"batch time\", t2 - t1)\n", 1137 | "\n", 1138 | "\n", 1139 | "await run()" 1140 | ] 1141 | }, 1142 | { 1143 | "cell_type": "code", 1144 | "execution_count": null, 1145 | "id": "9500b0f0-5866-4d5a-b744-ab96ed9a5d3e", 1146 | "metadata": {}, 1147 | "outputs": [], 1148 | "source": [] 1149 | }, 1150 | { 1151 | "cell_type": "code", 1152 | "execution_count": null, 1153 | "id": "a634d368-6fd1-4d8c-af27-252018cfb7bd", 1154 | "metadata": {}, 1155 | "outputs": [], 1156 | "source": [] 1157 | }, 1158 | { 1159 | "cell_type": "code", 1160 | "execution_count": null, 1161 | "id": "d810e1a7-84bf-47f5-a771-7ab582c61db0", 1162 | "metadata": {}, 1163 | "outputs": [], 1164 | "source": [] 1165 | }, 1166 | { 1167 | "cell_type": "code", 1168 | "execution_count": 33, 1169 | "id": "d83c1b23-1827-4543-8781-8b45e579b8d7", 1170 | "metadata": {}, 1171 | "outputs": [], 1172 | "source": [ 1173 | "import time\n", 1174 | "import asyncio\n", 1175 | "import aiohttp" 1176 | ] 1177 | }, 1178 | { 1179 | "cell_type": "code", 1180 | "execution_count": 34, 1181 | "id": "6f78dfd2-f741-4cf6-8408-1e76f6913268", 1182 | "metadata": {}, 1183 | "outputs": [], 1184 | "source": [ 1185 | "def do_timeit(coro):\n", 1186 | " async def inner(*args, **kwargs):\n", 1187 | " t1 = time.time()\n", 1188 | " try:\n", 1189 | " res = await coro(*args, **kwargs)\n", 1190 | " finally:\n", 1191 | " t2 = time.time()\n", 1192 | " print(f\"total time: {t2 - t1}\")\n", 1193 | "\n", 1194 | " return res\n", 1195 | "\n", 1196 | " return inner" 1197 | ] 1198 | }, 1199 | { 1200 | "cell_type": "code", 1201 | "execution_count": 37, 1202 | "id": "78006712-7485-47e0-bdb6-1dbf79205bb6", 1203 | "metadata": {}, 1204 | "outputs": [ 1205 | { 1206 | "name": "stdout", 1207 | "output_type": "stream", 1208 | "text": [ 1209 | "queue\n", 1210 | "workers\n", 1211 | "fetch_worker started worker_0\n", 1212 | "fetch_worker started worker_1\n", 1213 | "fetch_worker started worker_2\n", 1214 | "fetch_worker started worker_3\n", 1215 | "fetch_worker started worker_4\n", 1216 | "fetch_worker started worker_5\n", 1217 | "fetch_worker started worker_6\n", 1218 | "fetch_worker started worker_7\n", 1219 | "fetch_worker started worker_8\n", 1220 | "fetch_worker started worker_9\n", 1221 | "total time: 7.023946046829224\n" 1222 | ] 1223 | } 1224 | ], 1225 | "source": [ 1226 | "URL = \"https://docs.python.org/3/whatsnew/3.13.html\"\n", 1227 | "URLS = [URL] * 20\n", 1228 | "WORKERS = 10\n", 1229 | "\n", 1230 | "\n", 1231 | "async def fetch_url(session, sem, url):\n", 1232 | " async with sem:\n", 1233 | " await asyncio.sleep(1)\n", 1234 | " # async with session.get(url) as resp:\n", 1235 | " # return await resp.text()\n", 1236 | "\n", 1237 | "\n", 1238 | "async def fetch_worker(session, que, sem, name):\n", 1239 | " print(\"fetch_worker started\", name)\n", 1240 | " while True:\n", 1241 | " url = await que.get()\n", 1242 | "\n", 1243 | " try:\n", 1244 | " result = await fetch_url(session, sem, url)\n", 1245 | " except Exception as err:\n", 1246 | " print(\"error\", err)\n", 1247 | " finally:\n", 1248 | " que.task_done()\n", 1249 | "\n", 1250 | " print(\"fetch_worker finished\", name)\n", 1251 | "\n", 1252 | "\n", 1253 | "@do_timeit\n", 1254 | "async def run():\n", 1255 | " que = asyncio.Queue()\n", 1256 | " for url in URLS:\n", 1257 | " await que.put(url)\n", 1258 | "\n", 1259 | " print(\"queue\")\n", 1260 | "\n", 1261 | " sem = asyncio.Semaphore(3)\n", 1262 | "\n", 1263 | " async with aiohttp.ClientSession() as session:\n", 1264 | " workers = [\n", 1265 | " asyncio.create_task(\n", 1266 | " fetch_worker(session, que, sem, f\"worker_{i}\")\n", 1267 | " )\n", 1268 | " for i in range(WORKERS)\n", 1269 | " ]\n", 1270 | " print(\"workers\")\n", 1271 | "\n", 1272 | " await que.join()\n", 1273 | "\n", 1274 | " for worker in workers:\n", 1275 | " worker.cancel()\n", 1276 | "\n", 1277 | "\n", 1278 | "await run()" 1279 | ] 1280 | }, 1281 | { 1282 | "cell_type": "code", 1283 | "execution_count": null, 1284 | "id": "17a856b5-9106-4f41-8ede-c098514b2850", 1285 | "metadata": {}, 1286 | "outputs": [], 1287 | "source": [] 1288 | }, 1289 | { 1290 | "cell_type": "code", 1291 | "execution_count": null, 1292 | "id": "3406ded1-9caf-433c-bfa8-7a56da5b03ee", 1293 | "metadata": {}, 1294 | "outputs": [], 1295 | "source": [] 1296 | }, 1297 | { 1298 | "cell_type": "code", 1299 | "execution_count": 39, 1300 | "id": "66cdb38f-5a5d-481c-a0c0-c3ccacb10dec", 1301 | "metadata": {}, 1302 | "outputs": [ 1303 | { 1304 | "name": "stdout", 1305 | "output_type": "stream", 1306 | "text": [ 1307 | "start blocking api n=5\n", 1308 | "finish blocking api n=5\n", 1309 | "start non-blocking api n=5\n", 1310 | "finish non-blocking api n=5\n", 1311 | "total time: 10.006846189498901\n" 1312 | ] 1313 | } 1314 | ], 1315 | "source": [ 1316 | "async def bloking_api(n):\n", 1317 | " print(f\"start blocking api {n=}\")\n", 1318 | "\n", 1319 | " time.sleep(n)\n", 1320 | "\n", 1321 | " print(f\"finish blocking api {n=}\")\n", 1322 | "\n", 1323 | "\n", 1324 | "async def non_blocking_api(n):\n", 1325 | " print(f\"start non-blocking api {n=}\")\n", 1326 | "\n", 1327 | " await asyncio.sleep(n)\n", 1328 | " print(f\"finish non-blocking api {n=}\")\n", 1329 | "\n", 1330 | "\n", 1331 | "@do_timeit\n", 1332 | "async def run(n):\n", 1333 | " await asyncio.gather(bloking_api(n), non_blocking_api(n))\n", 1334 | "\n", 1335 | "\n", 1336 | "await run(5)" 1337 | ] 1338 | }, 1339 | { 1340 | "cell_type": "code", 1341 | "execution_count": 40, 1342 | "id": "db3e1414-e922-4191-9565-3ff0a49e565c", 1343 | "metadata": {}, 1344 | "outputs": [ 1345 | { 1346 | "name": "stdout", 1347 | "output_type": "stream", 1348 | "text": [ 1349 | "start blocking api n=5\n", 1350 | "start non-blocking api n=5\n", 1351 | "finish non-blocking api n=5\n", 1352 | "finish blocking api n=5\n", 1353 | "total time: 5.005144119262695\n" 1354 | ] 1355 | } 1356 | ], 1357 | "source": [ 1358 | "async def bloking_api(n):\n", 1359 | " print(f\"start blocking api {n=}\")\n", 1360 | "\n", 1361 | " await asyncio.to_thread(time.sleep, n)\n", 1362 | "\n", 1363 | " print(f\"finish blocking api {n=}\")\n", 1364 | "\n", 1365 | "\n", 1366 | "async def non_blocking_api(n):\n", 1367 | " print(f\"start non-blocking api {n=}\")\n", 1368 | "\n", 1369 | " await asyncio.sleep(n)\n", 1370 | " print(f\"finish non-blocking api {n=}\")\n", 1371 | "\n", 1372 | "\n", 1373 | "@do_timeit\n", 1374 | "async def run(n):\n", 1375 | " await asyncio.gather(bloking_api(n), non_blocking_api(n))\n", 1376 | "\n", 1377 | "\n", 1378 | "await run(5)" 1379 | ] 1380 | }, 1381 | { 1382 | "cell_type": "code", 1383 | "execution_count": 41, 1384 | "id": "1bea60a1-5e40-46bb-b09f-aa0206b38dcb", 1385 | "metadata": {}, 1386 | "outputs": [], 1387 | "source": [ 1388 | "import concurrent.futures" 1389 | ] 1390 | }, 1391 | { 1392 | "cell_type": "code", 1393 | "execution_count": 42, 1394 | "id": "c9081761-0d54-454a-b6ca-863e331fa980", 1395 | "metadata": {}, 1396 | "outputs": [ 1397 | { 1398 | "data": { 1399 | "text/plain": [ 1400 | "concurrent.futures._base.Future" 1401 | ] 1402 | }, 1403 | "execution_count": 42, 1404 | "metadata": {}, 1405 | "output_type": "execute_result" 1406 | } 1407 | ], 1408 | "source": [ 1409 | "concurrent.futures.Future" 1410 | ] 1411 | }, 1412 | { 1413 | "cell_type": "code", 1414 | "execution_count": 43, 1415 | "id": "cdc0fce4-011e-4d62-8943-25ec36b41ad9", 1416 | "metadata": {}, 1417 | "outputs": [], 1418 | "source": [ 1419 | "import asyncio.futures" 1420 | ] 1421 | }, 1422 | { 1423 | "cell_type": "code", 1424 | "execution_count": 44, 1425 | "id": "303a2bc6-56e3-4062-aade-c70a61f6664f", 1426 | "metadata": {}, 1427 | "outputs": [ 1428 | { 1429 | "data": { 1430 | "text/plain": [ 1431 | "False" 1432 | ] 1433 | }, 1434 | "execution_count": 44, 1435 | "metadata": {}, 1436 | "output_type": "execute_result" 1437 | } 1438 | ], 1439 | "source": [ 1440 | "concurrent.futures.Future is asyncio.futures.Future" 1441 | ] 1442 | }, 1443 | { 1444 | "cell_type": "code", 1445 | "execution_count": null, 1446 | "id": "9c793921-bd56-4378-8df6-00931f4e4824", 1447 | "metadata": {}, 1448 | "outputs": [], 1449 | "source": [] 1450 | }, 1451 | { 1452 | "cell_type": "code", 1453 | "execution_count": null, 1454 | "id": "00eeb733-bc9a-4eb8-926c-b73d8fc3f2ef", 1455 | "metadata": {}, 1456 | "outputs": [], 1457 | "source": [] 1458 | }, 1459 | { 1460 | "cell_type": "code", 1461 | "execution_count": 79, 1462 | "id": "e449f421-9c38-4812-92fe-8dc0f8a0b98f", 1463 | "metadata": {}, 1464 | "outputs": [ 1465 | { 1466 | "name": "stdout", 1467 | "output_type": "stream", 1468 | "text": [ 1469 | "start blocking api n=2\n", 1470 | "finish blocking api n=2\n", 1471 | "start non-blocking api n=2\n", 1472 | "finish non-blocking api n=2\n", 1473 | "total time: 3.9637069702148438\n" 1474 | ] 1475 | } 1476 | ], 1477 | "source": [ 1478 | "async def bloking_api(n):\n", 1479 | " print(f\"start blocking api {n=}\")\n", 1480 | "\n", 1481 | " res = factorial(350_000)\n", 1482 | "\n", 1483 | " print(f\"finish blocking api {n=}\")\n", 1484 | "\n", 1485 | "\n", 1486 | "async def non_blocking_api(n):\n", 1487 | " print(f\"start non-blocking api {n=}\")\n", 1488 | "\n", 1489 | " await asyncio.sleep(n)\n", 1490 | " print(f\"finish non-blocking api {n=}\")\n", 1491 | "\n", 1492 | "\n", 1493 | "@do_timeit\n", 1494 | "async def run(n):\n", 1495 | " await asyncio.gather(bloking_api(n), non_blocking_api(n))\n", 1496 | "\n", 1497 | "\n", 1498 | "await run(2)" 1499 | ] 1500 | }, 1501 | { 1502 | "cell_type": "code", 1503 | "execution_count": 80, 1504 | "id": "b67f6dec-d250-4433-bda5-e87a13f9c2b2", 1505 | "metadata": {}, 1506 | "outputs": [ 1507 | { 1508 | "name": "stdout", 1509 | "output_type": "stream", 1510 | "text": [ 1511 | "start blocking api n=2\n", 1512 | "start non-blocking api n=2\n", 1513 | "finish blocking api n=2\n", 1514 | "finish non-blocking api n=2\n", 1515 | "total time: 2.316499948501587\n" 1516 | ] 1517 | } 1518 | ], 1519 | "source": [ 1520 | "async def bloking_api(n):\n", 1521 | " print(f\"start blocking api {n=}\")\n", 1522 | "\n", 1523 | " await asyncio.to_thread(factorial, 350_000)\n", 1524 | "\n", 1525 | " print(f\"finish blocking api {n=}\")\n", 1526 | "\n", 1527 | "\n", 1528 | "async def non_blocking_api(n):\n", 1529 | " print(f\"start non-blocking api {n=}\")\n", 1530 | "\n", 1531 | " await asyncio.sleep(n)\n", 1532 | " print(f\"finish non-blocking api {n=}\")\n", 1533 | "\n", 1534 | "\n", 1535 | "@do_timeit\n", 1536 | "async def run(n):\n", 1537 | " await asyncio.gather(bloking_api(n), non_blocking_api(n))\n", 1538 | "\n", 1539 | "\n", 1540 | "await run(2)" 1541 | ] 1542 | }, 1543 | { 1544 | "cell_type": "code", 1545 | "execution_count": null, 1546 | "id": "f99235ce-c543-46ce-b20b-1894461097b5", 1547 | "metadata": {}, 1548 | "outputs": [], 1549 | "source": [] 1550 | } 1551 | ], 1552 | "metadata": { 1553 | "kernelspec": { 1554 | "display_name": "Python 3 (ipykernel)", 1555 | "language": "python", 1556 | "name": "python3" 1557 | }, 1558 | "language_info": { 1559 | "codemirror_mode": { 1560 | "name": "ipython", 1561 | "version": 3 1562 | }, 1563 | "file_extension": ".py", 1564 | "mimetype": "text/x-python", 1565 | "name": "python", 1566 | "nbconvert_exporter": "python", 1567 | "pygments_lexer": "ipython3", 1568 | "version": "3.13.2" 1569 | } 1570 | }, 1571 | "nbformat": 4, 1572 | "nbformat_minor": 5 1573 | } 1574 | -------------------------------------------------------------------------------- /lesson-07/homework.md: -------------------------------------------------------------------------------- 1 | # Домашнее задание #07 (async) 2 | 3 | ### 1. Скрипт для асинхронной обкачки урлов 4 | Написать скрипт для обкачки списка урлов из файла с возможностью задавать количество одновременных запросов, используя асинхронное программирование. 5 | Клиент можно использовать любой, например, из aiohttp. 6 | Например, 10 одновременных запросов могут задаваться командой: 7 | 8 | `python fetcher.py -c 10 urls.txt` 9 | или 10 | `python fetcher.py 10 urls.txt` 11 | 12 | Код должен быть выделен в функции и/или классы и методы. 13 | 14 | ### 2. Тесты в отдельном модуле 15 | 16 | ### 3. Зеленый пайплайн в репе 17 | Обязательно: тесты, покрытие, flake8, pylint. 18 | Опционально можно добавить другие инструменты, например, mypy и black. 19 | Покрытие тестов должно составлять не менее 90%. 20 | -------------------------------------------------------------------------------- /lesson-07/lesson-07.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mailcourses/deep_python_spring_2025/ede9fa206266b26d1da2c1857c8c41482677c3b9/lesson-07/lesson-07.pdf -------------------------------------------------------------------------------- /lesson-07/src/generator_socket.py: -------------------------------------------------------------------------------- 1 | # David Beazley algo 2 | import socket 3 | from select import select 4 | 5 | 6 | tasks = [] 7 | # sock: gen 8 | to_read = {} 9 | to_write = {} 10 | 11 | 12 | def server(): 13 | server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 14 | server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 15 | server_sock.bind(("localhost", 25000)) 16 | server_sock.listen() 17 | 18 | while True: 19 | yield "read", server_sock 20 | client_sock, addr = server_sock.accept() # read 21 | print("connect from", addr) 22 | 23 | tasks.append(client(client_sock)) 24 | 25 | 26 | def client(client_sock): 27 | while True: 28 | yield "read", client_sock 29 | data = client_sock.recv(4096) # read 30 | 31 | if not data: 32 | break 33 | else: 34 | yield "write", client_sock 35 | client_sock.send(data.decode().upper().encode()) # write 36 | 37 | client_sock.close() 38 | 39 | 40 | def event_loop(): 41 | while any([tasks, to_read, to_write]): 42 | 43 | while not tasks: 44 | ready_to_read, ready_to_write, _ = select(to_read, to_write, []) 45 | 46 | for sock in ready_to_read: 47 | tasks.append(to_read.pop(sock)) 48 | 49 | for sock in ready_to_write: 50 | tasks.append(to_write.pop(sock)) 51 | 52 | try: 53 | task = tasks.pop(0) 54 | op_type, sock = next(task) 55 | 56 | if op_type == "read": 57 | to_read[sock] = task 58 | elif op_type == "write": 59 | to_write[sock] = task 60 | 61 | except StopIteration: 62 | pass 63 | 64 | 65 | if __name__ == "__main__": 66 | tasks.append(server()) 67 | event_loop() 68 | -------------------------------------------------------------------------------- /lesson-07/src/select_socket.py: -------------------------------------------------------------------------------- 1 | import socket 2 | from select import select 3 | 4 | to_monitor = [] 5 | 6 | server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 7 | server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 8 | server_sock.bind(("localhost", 16000)) 9 | server_sock.listen(5) 10 | 11 | 12 | def accept_conn(server_sock): 13 | client_sock, addr = server_sock.accept() 14 | print("Connect", addr) 15 | to_monitor.append(client_sock) 16 | 17 | 18 | def respond(client_sock): 19 | data = client_sock.recv(4096) 20 | 21 | if data: 22 | client_sock.send(data.decode().upper().encode()) 23 | else: 24 | print("client closed") 25 | client_sock.close() 26 | to_monitor.remove(client_sock) 27 | 28 | 29 | def event_loop(): 30 | while True: 31 | ready_to_read, _, _ = select(to_monitor, [], []) # read, write, err 32 | for sock in ready_to_read: 33 | if sock is server_sock: 34 | accept_conn(sock) 35 | else: 36 | respond(sock) 37 | 38 | 39 | if __name__ == "__main__": 40 | to_monitor.append(server_sock) 41 | event_loop() 42 | -------------------------------------------------------------------------------- /lesson-07/src/selectors_socket.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import selectors 3 | 4 | selector = selectors.DefaultSelector() 5 | print("selector", selector) 6 | 7 | 8 | def server(): 9 | server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 10 | server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 11 | server_sock.bind(("localhost", 17000)) 12 | server_sock.listen(5) 13 | 14 | selector.register(server_sock, selectors.EVENT_READ, accept_conn) 15 | 16 | 17 | def accept_conn(server_sock): 18 | client_sock, addr = server_sock.accept() 19 | print("Connect", addr) 20 | selector.register(client_sock, selectors.EVENT_READ, respond) 21 | 22 | 23 | def respond(client_sock): 24 | data = client_sock.recv(4096) 25 | 26 | if data: 27 | client_sock.send(data.decode().upper().encode()) 28 | else: 29 | print("close client") 30 | selector.unregister(client_sock) 31 | client_sock.close() 32 | 33 | 34 | def event_loop(): 35 | while True: 36 | events = selector.select() # (key, events_mask) 37 | 38 | for key, _ in events: 39 | # key: NamedTuple(fileobj, events, data) 40 | callback = key.data 41 | callback(key.fileobj) 42 | 43 | 44 | if __name__ == "__main__": 45 | server() 46 | event_loop() 47 | -------------------------------------------------------------------------------- /lesson-07/src/socket_server.py: -------------------------------------------------------------------------------- 1 | import socket 2 | 3 | 4 | # server_sock = socket.socket() 5 | server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 6 | server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 7 | server_sock.bind(("localhost", 15000)) 8 | 9 | server_sock.listen(2) 10 | 11 | while True: 12 | client_sock, addr = server_sock.accept() 13 | print("client connected", addr) 14 | 15 | while True: 16 | # data = client_sock.recv(4) # http 17 | data = client_sock.recv(4096) 18 | if not data: 19 | break 20 | else: 21 | client_sock.send(data.decode().upper().encode()) 22 | 23 | client_sock.close() 24 | print("client gone:", addr) 25 | -------------------------------------------------------------------------------- /lesson-08/class_08.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "4d2d7f00-1570-44a7-be1e-b4bb5f6c7136", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import sys" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 3, 16 | "id": "bda787f1-7400-432f-8e11-71d162411e96", 17 | "metadata": {}, 18 | "outputs": [ 19 | { 20 | "data": { 21 | "text/plain": [ 22 | "(4294967295, 3)" 23 | ] 24 | }, 25 | "execution_count": 3, 26 | "metadata": {}, 27 | "output_type": "execute_result" 28 | } 29 | ], 30 | "source": [ 31 | "sys.getrefcount(1), sys.getrefcount(257)" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 4, 37 | "id": "ee03803e-6f8c-4627-9669-a1a2dee35f17", 38 | "metadata": {}, 39 | "outputs": [ 40 | { 41 | "data": { 42 | "text/plain": [ 43 | "2" 44 | ] 45 | }, 46 | "execution_count": 4, 47 | "metadata": {}, 48 | "output_type": "execute_result" 49 | } 50 | ], 51 | "source": [ 52 | "lst = [1, 2, 3]\n", 53 | "sys.getrefcount(lst) # obj = lst" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 5, 59 | "id": "fffe897d-163c-420f-b7b9-57321de6f811", 60 | "metadata": {}, 61 | "outputs": [], 62 | "source": [ 63 | "lst2 = lst" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": 6, 69 | "id": "099164b5-ef76-4601-a151-a79096c833f3", 70 | "metadata": {}, 71 | "outputs": [ 72 | { 73 | "data": { 74 | "text/plain": [ 75 | "3" 76 | ] 77 | }, 78 | "execution_count": 6, 79 | "metadata": {}, 80 | "output_type": "execute_result" 81 | } 82 | ], 83 | "source": [ 84 | "sys.getrefcount(lst)" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": 7, 90 | "id": "782b344e-fdab-4b75-970d-832d0924a44e", 91 | "metadata": {}, 92 | "outputs": [], 93 | "source": [ 94 | "del lst2" 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": 8, 100 | "id": "f72db34d-a1c0-4168-ae0c-820e021ea2ba", 101 | "metadata": {}, 102 | "outputs": [ 103 | { 104 | "data": { 105 | "text/plain": [ 106 | "2" 107 | ] 108 | }, 109 | "execution_count": 8, 110 | "metadata": {}, 111 | "output_type": "execute_result" 112 | } 113 | ], 114 | "source": [ 115 | "sys.getrefcount(lst)" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 19, 121 | "id": "0378df1b-e5f5-427d-abd1-b0bd0293b96e", 122 | "metadata": {}, 123 | "outputs": [], 124 | "source": [ 125 | "s1 = \"qwerty123\"" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": 20, 131 | "id": "63b66923-2608-48b2-abdf-7536b0f478bf", 132 | "metadata": {}, 133 | "outputs": [], 134 | "source": [ 135 | "s2 = \"qwerty123\"" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": 21, 141 | "id": "0495c3f2-9a8b-400c-af53-386dc969e458", 142 | "metadata": {}, 143 | "outputs": [ 144 | { 145 | "data": { 146 | "text/plain": [ 147 | "True" 148 | ] 149 | }, 150 | "execution_count": 21, 151 | "metadata": {}, 152 | "output_type": "execute_result" 153 | } 154 | ], 155 | "source": [ 156 | "s1 is s2" 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": 24, 162 | "id": "297fa2f1-1b1e-488c-9521-5bfd09f713c8", 163 | "metadata": {}, 164 | "outputs": [ 165 | { 166 | "data": { 167 | "text/plain": [ 168 | "(3, 3, 4294967295)" 169 | ] 170 | }, 171 | "execution_count": 24, 172 | "metadata": {}, 173 | "output_type": "execute_result" 174 | } 175 | ], 176 | "source": [ 177 | "sys.getrefcount(s1), sys.getrefcount(s2), sys.getrefcount(\"sys\")" 178 | ] 179 | }, 180 | { 181 | "cell_type": "code", 182 | "execution_count": null, 183 | "id": "4d0f9ff7-f678-410c-9e1a-969abd80288b", 184 | "metadata": {}, 185 | "outputs": [], 186 | "source": [] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": 12, 191 | "id": "70aeff39-9800-405f-b696-7a2530b5d8f0", 192 | "metadata": {}, 193 | "outputs": [], 194 | "source": [ 195 | "s1 = \"qwerty123!\"" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": 13, 201 | "id": "b77e395e-89ac-460b-ab80-ecaca91341c6", 202 | "metadata": {}, 203 | "outputs": [], 204 | "source": [ 205 | "s2 = \"qwerty123!\"" 206 | ] 207 | }, 208 | { 209 | "cell_type": "code", 210 | "execution_count": 14, 211 | "id": "795f6b4e-fde2-4f45-9810-6b1ed74e2b51", 212 | "metadata": {}, 213 | "outputs": [ 214 | { 215 | "data": { 216 | "text/plain": [ 217 | "False" 218 | ] 219 | }, 220 | "execution_count": 14, 221 | "metadata": {}, 222 | "output_type": "execute_result" 223 | } 224 | ], 225 | "source": [ 226 | "s1 is s2" 227 | ] 228 | }, 229 | { 230 | "cell_type": "code", 231 | "execution_count": 18, 232 | "id": "43e67065-253f-4c70-9ca3-24e281c496e6", 233 | "metadata": {}, 234 | "outputs": [ 235 | { 236 | "data": { 237 | "text/plain": [ 238 | "7" 239 | ] 240 | }, 241 | "execution_count": 18, 242 | "metadata": {}, 243 | "output_type": "execute_result" 244 | } 245 | ], 246 | "source": [ 247 | "sys.getrefcount(s1)" 248 | ] 249 | }, 250 | { 251 | "cell_type": "code", 252 | "execution_count": null, 253 | "id": "f0e78838-5906-476f-a448-42451eed15d7", 254 | "metadata": {}, 255 | "outputs": [], 256 | "source": [] 257 | }, 258 | { 259 | "cell_type": "code", 260 | "execution_count": null, 261 | "id": "65862096-cb0f-4d86-88e8-18fe3d2841dd", 262 | "metadata": {}, 263 | "outputs": [], 264 | "source": [] 265 | }, 266 | { 267 | "cell_type": "code", 268 | "execution_count": 25, 269 | "id": "96722924-dfad-4ce7-9a20-6c4d605d5156", 270 | "metadata": {}, 271 | "outputs": [ 272 | { 273 | "name": "stdout", 274 | "output_type": "stream", 275 | "text": [ 276 | "before 2\n", 277 | "sys.getrefcount(obj)=3\n", 278 | "after 2\n" 279 | ] 280 | } 281 | ], 282 | "source": [ 283 | "lst = [1, 2, 3]\n", 284 | "\n", 285 | "def count_refs(obj):\n", 286 | " print(f\"{sys.getrefcount(obj)=}\")\n", 287 | "\n", 288 | "print(\"before\", sys.getrefcount(lst))\n", 289 | "\n", 290 | "count_refs(lst)\n", 291 | "\n", 292 | "print(\"after\", sys.getrefcount(lst))" 293 | ] 294 | }, 295 | { 296 | "cell_type": "code", 297 | "execution_count": null, 298 | "id": "8c84b785-4e76-4756-a5a6-99917f899a2b", 299 | "metadata": {}, 300 | "outputs": [], 301 | "source": [] 302 | }, 303 | { 304 | "cell_type": "code", 305 | "execution_count": 26, 306 | "id": "fd3ce9af-f825-4af0-a1c5-5f864fde9e28", 307 | "metadata": {}, 308 | "outputs": [], 309 | "source": [ 310 | "class Animal:\n", 311 | " def __init__(self, name, food):\n", 312 | " self.name = name\n", 313 | " self.food = food\n", 314 | "\n", 315 | " def __str__(self):\n", 316 | " return f\"Animal({self.name}, {id(self)})\"\n", 317 | "\n", 318 | " def __del__(self):\n", 319 | " print(f\"del {str(self)=}\")" 320 | ] 321 | }, 322 | { 323 | "cell_type": "code", 324 | "execution_count": 33, 325 | "id": "e5124769-4f9f-491c-aac3-f78b5d4fdecd", 326 | "metadata": {}, 327 | "outputs": [ 328 | { 329 | "name": "stdout", 330 | "output_type": "stream", 331 | "text": [ 332 | "del str(self)='Animal(tiger, 4552684992)'\n", 333 | "created Animal(tiger, 4552685904)\n", 334 | "del str(self)='Animal(tiger, 4552685904)'\n" 335 | ] 336 | } 337 | ], 338 | "source": [ 339 | "tiger = Animal(\"tiger\", \"meat\")\n", 340 | "print(\"created\", tiger)\n", 341 | "\n", 342 | "sys.getrefcount(tiger)\n", 343 | "\n", 344 | "del tiger" 345 | ] 346 | }, 347 | { 348 | "cell_type": "code", 349 | "execution_count": null, 350 | "id": "851bccd7-e8d7-49a4-9dc1-8ef3ad565ac4", 351 | "metadata": {}, 352 | "outputs": [], 353 | "source": [] 354 | }, 355 | { 356 | "cell_type": "code", 357 | "execution_count": null, 358 | "id": "b7efb2dc-582c-42bb-a3e4-e326163a6b84", 359 | "metadata": {}, 360 | "outputs": [], 361 | "source": [] 362 | }, 363 | { 364 | "cell_type": "code", 365 | "execution_count": 35, 366 | "id": "731399aa-d700-4c4c-a102-238c67993557", 367 | "metadata": {}, 368 | "outputs": [ 369 | { 370 | "name": "stdout", 371 | "output_type": "stream", 372 | "text": [ 373 | "created Animal(tiger, 4552725088)\n", 374 | "created Animal(hog, 4551315536)\n", 375 | "3 3\n" 376 | ] 377 | } 378 | ], 379 | "source": [ 380 | "tiger = Animal(\"tiger\", \"meat\")\n", 381 | "hog = Animal(\"hog\", tiger)\n", 382 | "tiger.food = hog\n", 383 | "\n", 384 | "print(\"created\", tiger)\n", 385 | "print(\"created\", hog)\n", 386 | "\n", 387 | "print(sys.getrefcount(tiger), sys.getrefcount(tiger))" 388 | ] 389 | }, 390 | { 391 | "cell_type": "code", 392 | "execution_count": 49, 393 | "id": "dee74826-4bdd-4fd6-85bd-6a763ce6913a", 394 | "metadata": {}, 395 | "outputs": [ 396 | { 397 | "name": "stdout", 398 | "output_type": "stream", 399 | "text": [ 400 | "created Animal(tiger, 4559270896) {'name': 'tiger', 'food': <__main__.Animal object at 0x110136630>}\n", 401 | "created Animal(hog, 4564674096) {'name': 'hog', 'food': <__main__.Animal object at 0x10fc0f3f0>}\n", 402 | "3 3\n" 403 | ] 404 | } 405 | ], 406 | "source": [ 407 | "tiger = Animal(\"tiger\", \"meat\")\n", 408 | "hog = Animal(\"hog\", tiger)\n", 409 | "tiger.food = hog\n", 410 | "\n", 411 | "print(\"created\", tiger, tiger.__dict__)\n", 412 | "print(\"created\", hog, hog.__dict__)\n", 413 | "\n", 414 | "print(sys.getrefcount(tiger), sys.getrefcount(tiger))\n", 415 | "\n", 416 | "del tiger\n", 417 | "del hog" 418 | ] 419 | }, 420 | { 421 | "cell_type": "code", 422 | "execution_count": 37, 423 | "id": "cbd20273-44de-4084-becc-ffd8b4488ab7", 424 | "metadata": {}, 425 | "outputs": [], 426 | "source": [ 427 | "import gc" 428 | ] 429 | }, 430 | { 431 | "cell_type": "code", 432 | "execution_count": 42, 433 | "id": "57e1a8e6-2dc5-4839-b1dc-74d0fa0e7efe", 434 | "metadata": {}, 435 | "outputs": [ 436 | { 437 | "name": "stdout", 438 | "output_type": "stream", 439 | "text": [ 440 | "del str(self)='Animal(tiger, 4554609424)'\n", 441 | "del str(self)='Animal(hog, 4555459952)'\n" 442 | ] 443 | }, 444 | { 445 | "data": { 446 | "text/plain": [ 447 | "4455" 448 | ] 449 | }, 450 | "execution_count": 42, 451 | "metadata": {}, 452 | "output_type": "execute_result" 453 | } 454 | ], 455 | "source": [ 456 | "gc.collect()" 457 | ] 458 | }, 459 | { 460 | "cell_type": "code", 461 | "execution_count": null, 462 | "id": "afe88463-b681-48dd-8815-27a764e3c6fd", 463 | "metadata": {}, 464 | "outputs": [], 465 | "source": [] 466 | }, 467 | { 468 | "cell_type": "code", 469 | "execution_count": null, 470 | "id": "2349569a-9954-4379-96a9-e8f667ffaddd", 471 | "metadata": {}, 472 | "outputs": [], 473 | "source": [] 474 | }, 475 | { 476 | "cell_type": "code", 477 | "execution_count": null, 478 | "id": "a329a881-fe65-430f-9a1b-7d634d641c78", 479 | "metadata": {}, 480 | "outputs": [], 481 | "source": [] 482 | }, 483 | { 484 | "cell_type": "code", 485 | "execution_count": 39, 486 | "id": "c61fc358-c54e-462c-90c6-2e99d4deb774", 487 | "metadata": {}, 488 | "outputs": [], 489 | "source": [ 490 | "import ctypes" 491 | ] 492 | }, 493 | { 494 | "cell_type": "code", 495 | "execution_count": 40, 496 | "id": "c31f1bac-4d77-414d-85cd-27c112ac99ff", 497 | "metadata": {}, 498 | "outputs": [], 499 | "source": [ 500 | "class PyObject(ctypes.Structure):\n", 501 | " _fields_ = [(\"refcnt\", ctypes.c_long)]" 502 | ] 503 | }, 504 | { 505 | "cell_type": "code", 506 | "execution_count": null, 507 | "id": "b59595dd-0746-40cf-abf5-dd6c2123d3db", 508 | "metadata": {}, 509 | "outputs": [], 510 | "source": [] 511 | }, 512 | { 513 | "cell_type": "code", 514 | "execution_count": null, 515 | "id": "b04e0cbb-8db5-4d4e-a85b-7703b6c61efa", 516 | "metadata": {}, 517 | "outputs": [], 518 | "source": [] 519 | }, 520 | { 521 | "cell_type": "code", 522 | "execution_count": 47, 523 | "id": "5c51d83d-8b8e-400e-885c-fd7f28d9efea", 524 | "metadata": {}, 525 | "outputs": [ 526 | { 527 | "name": "stdout", 528 | "output_type": "stream", 529 | "text": [ 530 | "created Animal(tiger, 4554238016) 4554238016\n", 531 | "created Animal(hog, 4564596048) 4564596048\n", 532 | "3 3\n", 533 | "tiger: PyObject.from_address(addr).refcnt=2\n", 534 | "hog: PyObject.from_address(addr).refcnt=2\n", 535 | "tiger: PyObject.from_address(addr).refcnt=1\n", 536 | "hog: PyObject.from_address(addr).refcnt=1\n", 537 | "del str(self)='Animal(tiger, 4554238016)'\n", 538 | "del str(self)='Animal(hog, 4564596048)'\n", 539 | "tiger: PyObject.from_address(addr).refcnt=4515801808\n", 540 | "hog: PyObject.from_address(addr).refcnt=0\n" 541 | ] 542 | } 543 | ], 544 | "source": [ 545 | "tiger = Animal(\"tiger\", \"meat\")\n", 546 | "hog = Animal(\"hog\", tiger)\n", 547 | "tiger.food = hog\n", 548 | "\n", 549 | "id_tiger = id(tiger)\n", 550 | "id_hog = id(hog)\n", 551 | "\n", 552 | "print(\"created\", tiger, id_tiger)\n", 553 | "print(\"created\", hog, id_hog)\n", 554 | "\n", 555 | "print(sys.getrefcount(tiger), sys.getrefcount(tiger))\n", 556 | "\n", 557 | "for name, addr in ((\"tiger\", id_tiger), (\"hog\", id_hog)):\n", 558 | " print(f\"{name}: {PyObject.from_address(addr).refcnt=}\")\n", 559 | "\n", 560 | "del tiger\n", 561 | "del hog\n", 562 | "\n", 563 | "for name, addr in ((\"tiger\", id_tiger), (\"hog\", id_hog)):\n", 564 | " print(f\"{name}: {PyObject.from_address(addr).refcnt=}\")\n", 565 | "\n", 566 | "gc.collect()\n", 567 | "\n", 568 | "for name, addr in ((\"tiger\", id_tiger), (\"hog\", id_hog)):\n", 569 | " print(f\"{name}: {PyObject.from_address(addr).refcnt=}\")" 570 | ] 571 | }, 572 | { 573 | "cell_type": "code", 574 | "execution_count": null, 575 | "id": "fcc5db63-be34-4991-a05a-4261b76ad798", 576 | "metadata": {}, 577 | "outputs": [], 578 | "source": [] 579 | }, 580 | { 581 | "cell_type": "code", 582 | "execution_count": 46, 583 | "id": "121d7500-d2f3-41a9-843b-76ef92dd732e", 584 | "metadata": {}, 585 | "outputs": [ 586 | { 587 | "name": "stdout", 588 | "output_type": "stream", 589 | "text": [ 590 | "created Animal(tiger, 4554158192) 4554158192\n", 591 | "created Animal(hog, 4554156432) 4554156432\n", 592 | "3 3\n", 593 | "tiger: PyObject.from_address(addr).refcnt=2\n", 594 | "hog: PyObject.from_address(addr).refcnt=2\n", 595 | "tiger: PyObject.from_address(addr).refcnt=1\n", 596 | "hog: PyObject.from_address(addr).refcnt=1\n", 597 | "del str(self)='Animal(tiger, 4554158192)'\n", 598 | "del str(self)='Animal(hog, 4554156432)'\n", 599 | "tiger: PyObject.from_address(addr).refcnt=-1082298007296\n", 600 | "hog: PyObject.from_address(addr).refcnt=-16645888\n" 601 | ] 602 | } 603 | ], 604 | "source": [ 605 | "tiger = Animal(\"tiger\", \"meat\")\n", 606 | "hog = Animal(\"hog\", tiger)\n", 607 | "tiger.food = hog\n", 608 | "\n", 609 | "id_tiger = id(tiger)\n", 610 | "id_hog = id(hog)\n", 611 | "\n", 612 | "print(\"created\", tiger, id_tiger)\n", 613 | "print(\"created\", hog, id_hog)\n", 614 | "\n", 615 | "print(sys.getrefcount(tiger), sys.getrefcount(tiger))\n", 616 | "\n", 617 | "for name, addr in ((\"tiger\", id_tiger), (\"hog\", id_hog)):\n", 618 | " print(f\"{name}: {PyObject.from_address(addr).refcnt=}\")\n", 619 | "\n", 620 | "del tiger\n", 621 | "del hog\n", 622 | "\n", 623 | "for name, addr in ((\"tiger\", id_tiger), (\"hog\", id_hog)):\n", 624 | " print(f\"{name}: {PyObject.from_address(addr).refcnt=}\")\n", 625 | "\n", 626 | "gc.collect()\n", 627 | "\n", 628 | "for i in range(1000000):\n", 629 | " x = 1000\n", 630 | "\n", 631 | "for name, addr in ((\"tiger\", id_tiger), (\"hog\", id_hog)):\n", 632 | " print(f\"{name}: {PyObject.from_address(addr).refcnt=}\")" 633 | ] 634 | }, 635 | { 636 | "cell_type": "code", 637 | "execution_count": null, 638 | "id": "5c451e38-4db1-4fd6-814c-423f5e053c4d", 639 | "metadata": {}, 640 | "outputs": [], 641 | "source": [] 642 | }, 643 | { 644 | "cell_type": "code", 645 | "execution_count": null, 646 | "id": "7b303a18-0303-4155-a413-d085adb1cfea", 647 | "metadata": {}, 648 | "outputs": [], 649 | "source": [] 650 | }, 651 | { 652 | "cell_type": "code", 653 | "execution_count": 50, 654 | "id": "d3cbb578-a7e3-4c3b-a8c2-75297d025bf7", 655 | "metadata": {}, 656 | "outputs": [], 657 | "source": [ 658 | "import weakref" 659 | ] 660 | }, 661 | { 662 | "cell_type": "code", 663 | "execution_count": 51, 664 | "id": "241a6ae0-d8ec-442d-81f3-a125326b95ca", 665 | "metadata": {}, 666 | "outputs": [ 667 | { 668 | "name": "stdout", 669 | "output_type": "stream", 670 | "text": [ 671 | "1 2\n", 672 | "2 2\n", 673 | "\n", 674 | "3 3\n", 675 | "del str(self)='Animal(tiger, 4559536976)'\n", 676 | "tiger deleted\n", 677 | "4 del\n", 678 | "5 None\n" 679 | ] 680 | } 681 | ], 682 | "source": [ 683 | "tiger = Animal(\"tiger\", \"meat\")\n", 684 | "print(1, sys.getrefcount(tiger))\n", 685 | "\n", 686 | "weakref.finalize(tiger, lambda: print(\"tiger deleted\"))\n", 687 | "\n", 688 | "weak = weakref.ref(tiger)\n", 689 | "print(2, sys.getrefcount(tiger))\n", 690 | "\n", 691 | "print(weak)\n", 692 | "\n", 693 | "tiger_new = weak()\n", 694 | "print(3, sys.getrefcount(tiger))\n", 695 | "\n", 696 | "del tiger_new, tiger\n", 697 | "print(4, \"del\")\n", 698 | "\n", 699 | "tiger_new = weak()\n", 700 | "print(5, tiger_new)" 701 | ] 702 | }, 703 | { 704 | "cell_type": "code", 705 | "execution_count": null, 706 | "id": "bce1276c-4a22-4889-aee7-05c8dbc06421", 707 | "metadata": {}, 708 | "outputs": [], 709 | "source": [] 710 | }, 711 | { 712 | "cell_type": "code", 713 | "execution_count": null, 714 | "id": "97f6a2fd-50b2-44fb-9a19-1f53df5a3b45", 715 | "metadata": {}, 716 | "outputs": [], 717 | "source": [] 718 | }, 719 | { 720 | "cell_type": "code", 721 | "execution_count": null, 722 | "id": "ac41f7ad-2af4-4b64-b8d8-f30a550c151a", 723 | "metadata": {}, 724 | "outputs": [], 725 | "source": [] 726 | }, 727 | { 728 | "cell_type": "code", 729 | "execution_count": 67, 730 | "id": "fc9f0cba-9fbe-49fb-960b-eca1f44f31af", 731 | "metadata": {}, 732 | "outputs": [], 733 | "source": [ 734 | "class Point:\n", 735 | "\n", 736 | " def __init__(self, x, y):\n", 737 | " self.x = x\n", 738 | " self.y = y\n", 739 | " self.z = 33\n", 740 | "\n", 741 | "\n", 742 | "class Point2D(Point):\n", 743 | " __slots__ = (\"x\", \"y\")\n", 744 | "\n", 745 | " def __init__(self, x, y):\n", 746 | " #super().__init__(x, y)\n", 747 | " self.x = x\n", 748 | " self.y = y\n", 749 | " self.z = 33" 750 | ] 751 | }, 752 | { 753 | "cell_type": "code", 754 | "execution_count": 68, 755 | "id": "6d87c554-7632-488a-8301-9f5a07e814fe", 756 | "metadata": {}, 757 | "outputs": [], 758 | "source": [ 759 | "p = Point(10, 20)\n", 760 | "ps = Point2D(30, 40)" 761 | ] 762 | }, 763 | { 764 | "cell_type": "code", 765 | "execution_count": 65, 766 | "id": "dbf6f699-dc75-44d8-be56-7cc4685f7b1a", 767 | "metadata": {}, 768 | "outputs": [ 769 | { 770 | "data": { 771 | "text/plain": [ 772 | "{'x': 10, 'y': 20, 'z': 33}" 773 | ] 774 | }, 775 | "execution_count": 65, 776 | "metadata": {}, 777 | "output_type": "execute_result" 778 | } 779 | ], 780 | "source": [ 781 | "p.__dict__" 782 | ] 783 | }, 784 | { 785 | "cell_type": "code", 786 | "execution_count": 70, 787 | "id": "0cea3fc6-c613-47c2-8e88-4efc7d97b2e5", 788 | "metadata": {}, 789 | "outputs": [ 790 | { 791 | "data": { 792 | "text/plain": [ 793 | "{'z': 33, 'xxx': 34}" 794 | ] 795 | }, 796 | "execution_count": 70, 797 | "metadata": {}, 798 | "output_type": "execute_result" 799 | } 800 | ], 801 | "source": [ 802 | "ps.__dict__" 803 | ] 804 | }, 805 | { 806 | "cell_type": "code", 807 | "execution_count": null, 808 | "id": "508e0513-ad5a-4c61-9619-cf0ffa220b4b", 809 | "metadata": {}, 810 | "outputs": [], 811 | "source": [] 812 | }, 813 | { 814 | "cell_type": "code", 815 | "execution_count": null, 816 | "id": "6c82f479-e219-4e83-a1de-ab7f64f29f52", 817 | "metadata": {}, 818 | "outputs": [], 819 | "source": [] 820 | }, 821 | { 822 | "cell_type": "code", 823 | "execution_count": null, 824 | "id": "e4aebf4c-0b60-443a-8037-f5f513caf92d", 825 | "metadata": {}, 826 | "outputs": [], 827 | "source": [] 828 | }, 829 | { 830 | "cell_type": "code", 831 | "execution_count": 77, 832 | "id": "9be2537f-fb3d-4e95-a2b8-e520a72f4943", 833 | "metadata": {}, 834 | "outputs": [], 835 | "source": [ 836 | "class Point:\n", 837 | "\n", 838 | " def __init__(self, x, y):\n", 839 | " self.x = x\n", 840 | " self.y = y\n", 841 | "\n", 842 | "\n", 843 | "class Point2D:\n", 844 | " __slots__ = (\"x\", \"y\")\n", 845 | "\n", 846 | " def __init__(self, x, y):\n", 847 | " self.x = x\n", 848 | " self.y = y" 849 | ] 850 | }, 851 | { 852 | "cell_type": "code", 853 | "execution_count": 78, 854 | "id": "8cf0c1eb-6b38-44e1-b451-f6b03ca89afd", 855 | "metadata": {}, 856 | "outputs": [], 857 | "source": [ 858 | "p = Point(10, 20)\n", 859 | "ps = Point2D(30, 40)" 860 | ] 861 | }, 862 | { 863 | "cell_type": "code", 864 | "execution_count": 79, 865 | "id": "95a0139a-1eee-47de-bd0a-c7825819bb8b", 866 | "metadata": {}, 867 | "outputs": [ 868 | { 869 | "name": "stdout", 870 | "output_type": "stream", 871 | "text": [ 872 | "10 20 30 40\n" 873 | ] 874 | } 875 | ], 876 | "source": [ 877 | "print(p.x, p.y, ps.x, ps.y)" 878 | ] 879 | }, 880 | { 881 | "cell_type": "code", 882 | "execution_count": 80, 883 | "id": "9f6f18c0-671a-4720-b310-39b31223e68d", 884 | "metadata": {}, 885 | "outputs": [], 886 | "source": [ 887 | "p.x = 99\n", 888 | "ps.x = 42" 889 | ] 890 | }, 891 | { 892 | "cell_type": "code", 893 | "execution_count": 81, 894 | "id": "401720c9-3f7b-49d9-bade-9329f7e6b48f", 895 | "metadata": {}, 896 | "outputs": [ 897 | { 898 | "name": "stdout", 899 | "output_type": "stream", 900 | "text": [ 901 | "99 20 42 40\n" 902 | ] 903 | } 904 | ], 905 | "source": [ 906 | "print(p.x, p.y, ps.x, ps.y)" 907 | ] 908 | }, 909 | { 910 | "cell_type": "code", 911 | "execution_count": 82, 912 | "id": "4ab60dd3-a420-4288-a35c-7aeaa93b3561", 913 | "metadata": {}, 914 | "outputs": [], 915 | "source": [ 916 | "p.z = 25" 917 | ] 918 | }, 919 | { 920 | "cell_type": "code", 921 | "execution_count": 83, 922 | "id": "6f398a8b-fa39-43e2-9883-f6d29cbe4112", 923 | "metadata": {}, 924 | "outputs": [ 925 | { 926 | "ename": "AttributeError", 927 | "evalue": "'Point2D' object has no attribute 'z' and no __dict__ for setting new attributes", 928 | "output_type": "error", 929 | "traceback": [ 930 | "\u001b[31m---------------------------------------------------------------------------\u001b[39m", 931 | "\u001b[31mAttributeError\u001b[39m Traceback (most recent call last)", 932 | "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[83]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m \u001b[43mps\u001b[49m\u001b[43m.\u001b[49m\u001b[43mz\u001b[49m = \u001b[32m25\u001b[39m\n", 933 | "\u001b[31mAttributeError\u001b[39m: 'Point2D' object has no attribute 'z' and no __dict__ for setting new attributes" 934 | ] 935 | } 936 | ], 937 | "source": [ 938 | "ps.z = 25" 939 | ] 940 | }, 941 | { 942 | "cell_type": "code", 943 | "execution_count": 85, 944 | "id": "792b8a54-213c-4d1f-a82a-983efcd90d64", 945 | "metadata": {}, 946 | "outputs": [ 947 | { 948 | "data": { 949 | "text/plain": [ 950 | "mappingproxy({'__module__': '__main__',\n", 951 | " '__firstlineno__': 1,\n", 952 | " '__init__': ,\n", 953 | " '__static_attributes__': ('x', 'y'),\n", 954 | " '__dict__': ,\n", 955 | " '__weakref__': ,\n", 956 | " '__doc__': None})" 957 | ] 958 | }, 959 | "execution_count": 85, 960 | "metadata": {}, 961 | "output_type": "execute_result" 962 | } 963 | ], 964 | "source": [ 965 | "Point.__dict__" 966 | ] 967 | }, 968 | { 969 | "cell_type": "code", 970 | "execution_count": 86, 971 | "id": "4c9a888e-4673-4523-90c3-e03581b1ff8c", 972 | "metadata": {}, 973 | "outputs": [ 974 | { 975 | "data": { 976 | "text/plain": [ 977 | "mappingproxy({'__module__': '__main__',\n", 978 | " '__firstlineno__': 8,\n", 979 | " '__slots__': ('x', 'y'),\n", 980 | " '__init__': ,\n", 981 | " '__static_attributes__': ('x', 'y'),\n", 982 | " 'x': ,\n", 983 | " 'y': ,\n", 984 | " '__doc__': None})" 985 | ] 986 | }, 987 | "execution_count": 86, 988 | "metadata": {}, 989 | "output_type": "execute_result" 990 | } 991 | ], 992 | "source": [ 993 | "Point2D.__dict__" 994 | ] 995 | }, 996 | { 997 | "cell_type": "code", 998 | "execution_count": 88, 999 | "id": "4ed3d357-4af5-4e3f-be6e-6b80b71cb146", 1000 | "metadata": {}, 1001 | "outputs": [], 1002 | "source": [ 1003 | "def create_objs(cls, n):\n", 1004 | " objs = [cls(2 ** 20, 2 ** 21) for _ in range(n)]\n", 1005 | " for obj in objs:\n", 1006 | " obj.x = obj.x + obj.y\n", 1007 | " obj.y = obj.x - obj.y\n", 1008 | " obj.x = obj.x + obj.x + obj.x - obj.y" 1009 | ] 1010 | }, 1011 | { 1012 | "cell_type": "code", 1013 | "execution_count": 90, 1014 | "id": "2630d7dd-2a6f-4d90-a459-e9414ce397b3", 1015 | "metadata": {}, 1016 | "outputs": [ 1017 | { 1018 | "name": "stdout", 1019 | "output_type": "stream", 1020 | "text": [ 1021 | "CPU times: user 5.39 s, sys: 534 ms, total: 5.93 s\n", 1022 | "Wall time: 7.06 s\n" 1023 | ] 1024 | } 1025 | ], 1026 | "source": [ 1027 | "%%time\n", 1028 | "\n", 1029 | "create_objs(Point, 5_000_000)" 1030 | ] 1031 | }, 1032 | { 1033 | "cell_type": "code", 1034 | "execution_count": 91, 1035 | "id": "1ef907e4-7c11-4180-bdd9-da4ec55a4ad1", 1036 | "metadata": {}, 1037 | "outputs": [ 1038 | { 1039 | "name": "stdout", 1040 | "output_type": "stream", 1041 | "text": [ 1042 | "CPU times: user 4.83 s, sys: 440 ms, total: 5.27 s\n", 1043 | "Wall time: 6.49 s\n" 1044 | ] 1045 | } 1046 | ], 1047 | "source": [ 1048 | "%%time\n", 1049 | "\n", 1050 | "create_objs(Point2D, 5_000_000)" 1051 | ] 1052 | }, 1053 | { 1054 | "cell_type": "code", 1055 | "execution_count": null, 1056 | "id": "7f64eef8-b0b5-4099-9e61-87b9405e6452", 1057 | "metadata": {}, 1058 | "outputs": [], 1059 | "source": [] 1060 | }, 1061 | { 1062 | "cell_type": "code", 1063 | "execution_count": null, 1064 | "id": "1e07c3e0-10b6-490e-a166-574fe17e5a2a", 1065 | "metadata": {}, 1066 | "outputs": [], 1067 | "source": [] 1068 | }, 1069 | { 1070 | "cell_type": "code", 1071 | "execution_count": null, 1072 | "id": "1510ca19-56c7-42c1-92a8-26c422ad81c8", 1073 | "metadata": {}, 1074 | "outputs": [], 1075 | "source": [] 1076 | }, 1077 | { 1078 | "cell_type": "code", 1079 | "execution_count": 1, 1080 | "id": "2ec9764d-e9a2-4378-b968-3f88032dae10", 1081 | "metadata": {}, 1082 | "outputs": [], 1083 | "source": [ 1084 | "class Point:\n", 1085 | "\n", 1086 | " def __init__(self, x, y):\n", 1087 | " self.x = x\n", 1088 | " self.y = y\n", 1089 | "\n", 1090 | "\n", 1091 | "class Point2D(Point):\n", 1092 | " __slots__ = (\"x\", \"y\")\n", 1093 | "\n", 1094 | " def __init__(self, x, y):\n", 1095 | " self.x = x\n", 1096 | " self.y = y" 1097 | ] 1098 | }, 1099 | { 1100 | "cell_type": "code", 1101 | "execution_count": 2, 1102 | "id": "c5908ef1-3d34-4356-901d-5492c9251c33", 1103 | "metadata": {}, 1104 | "outputs": [], 1105 | "source": [ 1106 | "p = Point2D(10, 20)" 1107 | ] 1108 | }, 1109 | { 1110 | "cell_type": "code", 1111 | "execution_count": 3, 1112 | "id": "5859e158-ab10-4fb4-a9b9-57a16ac1a855", 1113 | "metadata": {}, 1114 | "outputs": [ 1115 | { 1116 | "data": { 1117 | "text/plain": [ 1118 | "{}" 1119 | ] 1120 | }, 1121 | "execution_count": 3, 1122 | "metadata": {}, 1123 | "output_type": "execute_result" 1124 | } 1125 | ], 1126 | "source": [ 1127 | "p.__dict__" 1128 | ] 1129 | }, 1130 | { 1131 | "cell_type": "code", 1132 | "execution_count": 4, 1133 | "id": "0fa0a258-b7fb-4213-acd1-732e87ecbdec", 1134 | "metadata": {}, 1135 | "outputs": [], 1136 | "source": [ 1137 | "import sys" 1138 | ] 1139 | }, 1140 | { 1141 | "cell_type": "code", 1142 | "execution_count": 5, 1143 | "id": "b7ae2640-101d-40e7-b948-5a32e92e9bce", 1144 | "metadata": {}, 1145 | "outputs": [ 1146 | { 1147 | "data": { 1148 | "text/plain": [ 1149 | "64" 1150 | ] 1151 | }, 1152 | "execution_count": 5, 1153 | "metadata": {}, 1154 | "output_type": "execute_result" 1155 | } 1156 | ], 1157 | "source": [ 1158 | "sys.getsizeof(p)" 1159 | ] 1160 | }, 1161 | { 1162 | "cell_type": "code", 1163 | "execution_count": 6, 1164 | "id": "b781fd82-8cb4-4a94-b05b-6b9b4c3605aa", 1165 | "metadata": {}, 1166 | "outputs": [ 1167 | { 1168 | "data": { 1169 | "text/plain": [ 1170 | "304" 1171 | ] 1172 | }, 1173 | "execution_count": 6, 1174 | "metadata": {}, 1175 | "output_type": "execute_result" 1176 | } 1177 | ], 1178 | "source": [ 1179 | "sys.getsizeof(p.__dict__)" 1180 | ] 1181 | }, 1182 | { 1183 | "cell_type": "code", 1184 | "execution_count": 7, 1185 | "id": "4b4c78ea-22bf-4ffc-873f-313c881fafbf", 1186 | "metadata": {}, 1187 | "outputs": [ 1188 | { 1189 | "data": { 1190 | "text/plain": [ 1191 | "64" 1192 | ] 1193 | }, 1194 | "execution_count": 7, 1195 | "metadata": {}, 1196 | "output_type": "execute_result" 1197 | } 1198 | ], 1199 | "source": [ 1200 | "sys.getsizeof({})" 1201 | ] 1202 | }, 1203 | { 1204 | "cell_type": "code", 1205 | "execution_count": 8, 1206 | "id": "af7181af-a875-4a70-9ea9-e9e8648d446d", 1207 | "metadata": {}, 1208 | "outputs": [ 1209 | { 1210 | "data": { 1211 | "text/plain": [ 1212 | "224" 1213 | ] 1214 | }, 1215 | "execution_count": 8, 1216 | "metadata": {}, 1217 | "output_type": "execute_result" 1218 | } 1219 | ], 1220 | "source": [ 1221 | "sys.getsizeof({1: 11})" 1222 | ] 1223 | }, 1224 | { 1225 | "cell_type": "code", 1226 | "execution_count": 9, 1227 | "id": "a0f53409-1030-4c9f-bc6b-fcc7b39853f5", 1228 | "metadata": {}, 1229 | "outputs": [ 1230 | { 1231 | "name": "stdout", 1232 | "output_type": "stream", 1233 | "text": [ 1234 | "0 224 1\n", 1235 | "1 224 2\n", 1236 | "2 224 3\n", 1237 | "3 224 4\n", 1238 | "4 224 5\n", 1239 | "5 352 6\n", 1240 | "6 352 7\n", 1241 | "7 352 8\n" 1242 | ] 1243 | } 1244 | ], 1245 | "source": [ 1246 | "dct = {}\n", 1247 | "\n", 1248 | "for i in range(8):\n", 1249 | " dct[i] = i\n", 1250 | " print(i, sys.getsizeof(dct), len(dct))" 1251 | ] 1252 | }, 1253 | { 1254 | "cell_type": "code", 1255 | "execution_count": null, 1256 | "id": "ff27c69c-2676-434f-9e3e-e5b24f85c218", 1257 | "metadata": {}, 1258 | "outputs": [], 1259 | "source": [] 1260 | }, 1261 | { 1262 | "cell_type": "code", 1263 | "execution_count": null, 1264 | "id": "cdbf8a5e-c45a-4b1e-90b3-3ab2210258a7", 1265 | "metadata": {}, 1266 | "outputs": [], 1267 | "source": [] 1268 | } 1269 | ], 1270 | "metadata": { 1271 | "kernelspec": { 1272 | "display_name": "Python 3 (ipykernel)", 1273 | "language": "python", 1274 | "name": "python3" 1275 | }, 1276 | "language_info": { 1277 | "codemirror_mode": { 1278 | "name": "ipython", 1279 | "version": 3 1280 | }, 1281 | "file_extension": ".py", 1282 | "mimetype": "text/x-python", 1283 | "name": "python", 1284 | "nbconvert_exporter": "python", 1285 | "pygments_lexer": "ipython3", 1286 | "version": "3.13.2" 1287 | } 1288 | }, 1289 | "nbformat": 4, 1290 | "nbformat_minor": 5 1291 | } 1292 | -------------------------------------------------------------------------------- /lesson-08/homework.md: -------------------------------------------------------------------------------- 1 | # Домашнее задание #08 (память, профилирование) 2 | 3 | ### 1. Сравнение использования weakref и слотов 4 | Нужно придумать свои типы с несколькими атрибутами: 5 | - класс с обычными атрибутами 6 | - класс со слотами 7 | - класс с атрибутами weakref 8 | 9 | Для каждого класса создается большое число экземпляров и замеряется (сравнивается): 10 | - время создания пачки экземпляров; 11 | - время чтения/изменения атрибутов пачки экземпляров. 12 | 13 | Результаты замеров оформляются скриншотами c описанием и выводом. 14 | 15 | ### 2. Профилирование 16 | Провести профилирование вызовов и памяти для кода из пункта 1. 17 | 18 | Результаты оформляются скриншотами c описанием и выводом. 19 | 20 | ### 3. Декоратор для профилирования 21 | Применение декоратора к функции должно выполнять прoфилирование (cProfile) всех вызовов данной функции. 22 | Вызов метода `.print_stat()` должен выводить единую таблицу со статистикой профилирования суммарно по всем вызовам функции. 23 | 24 | 25 | ```py 26 | def profile_deco(...): 27 | ... 28 | 29 | 30 | @profile_deco 31 | def add(a, b): 32 | return a + b 33 | 34 | 35 | @profile_deco 36 | def sub(a, b): 37 | return a - b 38 | 39 | 40 | add(1, 2) 41 | add(4, 5) 42 | sub(4, 5) 43 | 44 | 45 | add.print_stat() # выводится результат профилирования суммарно по всем вызовам функции add (всего два вызова) 46 | sub.print_stat() # выводится результат профилирования суммарно по всем вызовам функции sub (всего один вызов) 47 | ``` 48 | 49 | ### 4. Зеленый пайплайн в репе 50 | Обязательно: flake8, pylint. 51 | Опционально можно добавить другие инструменты, например, mypy и black. 52 | -------------------------------------------------------------------------------- /lesson-08/lesson-08.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mailcourses/deep_python_spring_2025/ede9fa206266b26d1da2c1857c8c41482677c3b9/lesson-08/lesson-08.pdf -------------------------------------------------------------------------------- /lesson-09/homework.md: -------------------------------------------------------------------------------- 1 | # Домашнее задание #09 (логирование) 2 | 3 | ### 1. Логирование LRUCache (hw #05) 4 | Код решения должен быть целиком в каталоге данного ДЗ без ссылок/импортов на домашки про LRUCache. 5 | Корректность LRUCache в данном задании не проверяется. 6 | 7 | - Нужно добавить логирование разного уровня в файл cache.log. 8 | - По аргументу командной строки "-s" дополнительно логировать в stdout с отдельным форматированием. 9 | - По аргументу командной строки "-f" нужно применять кастомный фильтр, например, отбрасывающий записи c четным числом слов или что-то свое. 10 | - "-s" и "-f" могут указываеться в одном запуске и должны работать вместе в таком случае (модуль argparse). 11 | 12 | Логирование должно покрывать как минимум следующие случаи: 13 | - get существующего ключа 14 | - get отсутствующего ключа 15 | - set отсутствующего ключа 16 | - set отсутствующего ключа, когда достигнута ёмкость 17 | - set существующего ключа 18 | - различные debug записи в дополнение и в зависимости от реализации 19 | 20 | При запуске модуля должны выполняться все перечисленные операции с кэшом (через функцию в `if __name__ == "__main__"`). 21 | 22 | ### 2. Зеленый пайплайн в репе 23 | Обязательно: flake8, pylint. 24 | Опционально можно добавить другие инструменты, например, mypy и black. 25 | -------------------------------------------------------------------------------- /lesson-09/lesson-09.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mailcourses/deep_python_spring_2025/ede9fa206266b26d1da2c1857c8c41482677c3b9/lesson-09/lesson-09.pdf -------------------------------------------------------------------------------- /lesson-10/homework.md: -------------------------------------------------------------------------------- 1 | # Домашнее задание #10 (Расширения на C) 2 | 3 | ### 1. Реализовать библиотеку для парсинга и сериализации json (с помощью C API) 4 | [!] Если реализация на C совсем не поддается, то можно реализовать на Python и скоппилировать через Cython. 5 | 6 | - Нужно написать модуль custom_json, который имел бы хотя бы два метода: loads и dumps; 7 | - Методу loads на вход подаётся строка в формате JSON. Ограничения: 8 | * JSON-сообщение в виде набор пар ключ-значение (читай как python-словарь); 9 | * Ключём в JSON **всегда** является строка в двойных кавычках; 10 | * Значением может выступать либо число, либо строка. Если захотелось приключений, то можно сделать поддержку и других типов; 11 | * Если входная строка не является JSON-объектом, то выбрасывается исключение 12 | ```C 13 | ... 14 | PyErr_Format(PyExc_TypeError, "Expected object or value"); 15 | return NULL; 16 | ``` 17 | * Возвращаться должен объект типа dict. Например, можно сделать так: 18 | ```C 19 | PyObject *dict = NULL; 20 | if (!(dict = PyDict_New())) { 21 | printf("ERROR: Failed to create Dict Object\n"); 22 | return NULL; 23 | } 24 | 25 | PyObject *key = NULL; 26 | PyObject *value = NULL; 27 | 28 | if (!(key = Py_BuildValue("s", "hello"))) { 29 | printf("ERROR: Failed to build string value\n"); 30 | return NULL; 31 | } 32 | if (!(value = Py_BuildValue("i", 10))) { 33 | printf("ERROR: Failed to build integer value\n"); 34 | return NULL; 35 | } 36 | if (PyDict_SetItem(dict, key, value) < 0) { 37 | printf("ERROR: Failed to set item\n"); 38 | return NULL; 39 | } 40 | if (!(key = Py_BuildValue("s", "world"))) { 41 | printf("ERROR: Failed to build string value\n"); 42 | return NULL; 43 | } 44 | if (!(value = Py_BuildValue("s", "100500"))) { 45 | printf("ERROR: Failed to build string value\n"); 46 | return NULL; 47 | } 48 | if (PyDict_SetItem(dict, key, value) < 0) { 49 | printf("ERROR: Failed to set item\n"); 50 | return NULL; 51 | } 52 | 53 | return dict; 54 | ``` 55 | - Методу dumps в качестве аргумента передаётся объект типа dict и возвращает строку. Ограничения как у loads только наоборот; 56 | 57 | Готовое расширение используется из Python: 58 | ```Python 59 | import json 60 | 61 | import custom_json 62 | 63 | 64 | def main(): 65 | json_str = '{"hello": 10, "world": "value"}' 66 | 67 | json_doc = json.loads(json_str) 68 | cust_json_doc = custom_json.loads(json_str) 69 | 70 | assert json_doc == cust_json_doc 71 | assert json_str == cust_json.dumps(cust_json.loads(json_str)) 72 | 73 | 74 | if __name__ == "__main__": 75 | main() 76 | ``` 77 | 78 | ### 2. Тесты корректности на уровне Python в отдельном модуле 79 | 80 | ### 3. Тесты производительности 81 | Сравнивать скорость работы своей реализации с json на одних и тех же данных. 82 | Данные должны быть большие (как количество JSON, так и размер каждого JSON). Требование: выполнение тестов не менее 100 мс. 83 | 84 | Для генерации тестовых JSON можно использовать статический найденный на просторах интернета JSON. 85 | Можно попробовать использовать библиотеку [Faker](https://faker.readthedocs.io/en/master/) для генерации данных. 86 | Допустимо генерировать рандомные данные. 87 | 88 | ### 4. Зеленый пайплайн в репе 89 | Обязательно: тесты, покрытие, flake8, pylint. 90 | Опционально можно добавить другие инструменты, например, mypy и black. 91 | Покрытие тестов должно составлять не менее 90%. 92 | -------------------------------------------------------------------------------- /lesson-10/lesson-10.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mailcourses/deep_python_spring_2025/ede9fa206266b26d1da2c1857c8c41482677c3b9/lesson-10/lesson-10.pdf -------------------------------------------------------------------------------- /lesson-10/src/fib_capi/fibutils_capi.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | 6 | 7 | int fib_rec_c_impl(int n) 8 | { 9 | if (n < 3) 10 | return 1; 11 | 12 | return fib_rec_c_impl(n - 1) + fib_rec_c_impl(n - 2); 13 | } 14 | 15 | 16 | int fib_iter_c_impl(int n) 17 | { 18 | int a = 0, b = 1; 19 | for (int i = 0; i < n; ++i) 20 | { 21 | int tmp = b; 22 | b = a + b; 23 | a = tmp; 24 | } 25 | return a; 26 | } 27 | 28 | 29 | PyObject *fibutils_capi_fib_rec_c(PyObject* self, PyObject* args) 30 | { 31 | int n; 32 | if (!PyArg_ParseTuple(args, "i", &n)) 33 | return NULL; 34 | 35 | int res = fib_rec_c_impl(n); 36 | return PyLong_FromLong(res); 37 | } 38 | 39 | 40 | PyObject *fibutils_capi_fib_iter_c(PyObject* self, PyObject* args) 41 | { 42 | int n; 43 | if (!PyArg_ParseTuple(args, "i", &n)) 44 | return NULL; 45 | 46 | int res = fib_iter_c_impl(n); 47 | return PyLong_FromLong(res); 48 | } 49 | 50 | 51 | static PyMethodDef methods[] = { 52 | {"fib_rec_c", fibutils_capi_fib_rec_c, METH_VARARGS, "recursive fib with C-api"}, 53 | {"fib_iter_c", fibutils_capi_fib_iter_c, METH_VARARGS, "iter fib with C-api"}, 54 | {NULL, NULL, 0, NULL}, 55 | }; 56 | 57 | 58 | static struct PyModuleDef module_fibutils_capi = { 59 | PyModuleDef_HEAD_INIT, "fibutils_capi", NULL, -1, methods 60 | }; 61 | 62 | 63 | PyMODINIT_FUNC PyInit_fibutils_capi() 64 | { 65 | return PyModule_Create( &module_fibutils_capi ); 66 | } 67 | -------------------------------------------------------------------------------- /lesson-10/src/fib_capi/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, Extension 2 | 3 | 4 | def main(): 5 | setup( 6 | name="fibutils_capi", 7 | version="1.0.0", 8 | ext_modules=[Extension("fibutils_capi", ["fibutils_capi.c"])], 9 | ) 10 | 11 | 12 | if __name__ == "__main__": 13 | main() 14 | -------------------------------------------------------------------------------- /lesson-10/src/fib_cffi/test_cffi.py: -------------------------------------------------------------------------------- 1 | import cffi 2 | 3 | 4 | def test_sum(): 5 | ffi = cffi.FFI() 6 | lib = ffi.dlopen("../fib_ctypes/libfibutils.so") 7 | 8 | ffi.cdef("int sum(int *arr, int len);") 9 | 10 | lst = list(range(10, 20)) 11 | len_lst = len(lst) 12 | 13 | arr = ffi.new("int[]", lst) 14 | print(arr) 15 | 16 | res = lib.sum(arr, len_lst) 17 | print(f"{res=}, {sum(lst)=}, {type(res)}") 18 | 19 | 20 | def test_build(): 21 | builder = cffi.FFI() 22 | builder.cdef("int mult(int a, int b, int c);") 23 | 24 | builder.set_source( 25 | "tmp_mult", 26 | """ 27 | int mult(int a, int b, int c) 28 | { 29 | return a * b * c; 30 | } 31 | """ 32 | ) 33 | builder.compile() 34 | 35 | print("compiled!") 36 | 37 | from tmp_mult import lib 38 | 39 | a, b, c = 2, 10, 4 40 | res = lib.mult(a, b, c) 41 | 42 | print(f"{res=}, {a * b * c=}") 43 | 44 | 45 | if __name__ == "__main__": 46 | test_sum() 47 | test_build() 48 | -------------------------------------------------------------------------------- /lesson-10/src/fib_ctypes/fibutils.c: -------------------------------------------------------------------------------- 1 | int fib_rec_ctypes(int n) 2 | { 3 | if (n < 3) 4 | return 1; 5 | 6 | return fib_rec_ctypes(n - 1) + fib_rec_ctypes(n - 2); 7 | } 8 | 9 | 10 | int fib_iter_ctypes(int n) 11 | { 12 | int a = 0, b = 1; 13 | for (int i = 0; i < n; ++i) 14 | { 15 | int tmp = b; 16 | b = a + b; 17 | a = tmp; 18 | } 19 | return a; 20 | } 21 | 22 | 23 | int sum(int *arr, int len) 24 | { 25 | int res = 0; 26 | for (int i = 0; i < len; ++i) 27 | { 28 | res += arr[i]; 29 | } 30 | return res; 31 | } 32 | -------------------------------------------------------------------------------- /lesson-10/src/fib_ctypes/test_fibutils.py: -------------------------------------------------------------------------------- 1 | import ctypes 2 | 3 | 4 | def test_sum(): 5 | lib = ctypes.cdll.LoadLibrary("./libfibutils.so") 6 | lib.sum.argstype = [ctypes.POINTER(ctypes.c_int), ctypes.c_int] 7 | lib.sum.restype = ctypes.c_int 8 | 9 | lst = list(range(10, 20)) 10 | lst = [2 ** 99, 1, 32] 11 | len_lst = len(lst) 12 | 13 | arr_int_type = ctypes.c_int * len_lst 14 | res = int(lib.sum(arr_int_type(*lst), ctypes.c_int(len_lst))) 15 | 16 | print(f"{res=}, {sum(lst)=}") 17 | 18 | 19 | def test_strstr(): 20 | lib = ctypes.CDLL(None) 21 | lib.strstr.argstype = [ctypes.c_char_p, ctypes.c_char_p] 22 | lib.strstr.restype = ctypes.c_char_p 23 | 24 | print(f"{lib.strstr(b'qwerty', b'er')=}") 25 | print(f"{lib.strstr(b'qwerty', b'sd')=}") 26 | print(f"{lib.strstr(b'qwerty', b'ty')=}") 27 | 28 | 29 | if __name__ == "__main__": 30 | test_sum() 31 | test_strstr() 32 | -------------------------------------------------------------------------------- /lesson-10/src/fib_cython/fibcyth.pyx: -------------------------------------------------------------------------------- 1 | cpdef int fib_rec_cy(int n): 2 | if n < 3: 3 | return 1 4 | 5 | return fib_rec_cy(n - 1) + fib_rec_cy(n - 2) 6 | 7 | 8 | cpdef int fib_iter_cy(int n): 9 | cdef int a = 0 10 | cdef int b = 1 11 | 12 | for _ in range(n): 13 | a, b = b, a + b 14 | 15 | return a 16 | -------------------------------------------------------------------------------- /lesson-10/src/fib_cython/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | from Cython.Build import cythonize 3 | 4 | 5 | setup( 6 | ext_modules= cythonize(["fibcyth.pyx"]) 7 | ) 8 | -------------------------------------------------------------------------------- /lesson-10/src/fib_native.py: -------------------------------------------------------------------------------- 1 | 2 | def fib_rec_py(n: int) -> int: 3 | if n < 3: 4 | return 1 5 | 6 | return fib_rec_py(n - 1) + fib_rec_py(n - 2) 7 | 8 | 9 | def fib_iter_py(n: int) -> int: 10 | a, b = 0, 1 11 | 12 | for _ in range(n): 13 | a, b = b, a + b 14 | 15 | return a 16 | -------------------------------------------------------------------------------- /lesson-10/src/perf.py: -------------------------------------------------------------------------------- 1 | import ctypes 2 | import time 3 | 4 | import cffi 5 | from fib_native import fib_rec_py, fib_iter_py 6 | import fibutils_capi 7 | import fibcyth 8 | 9 | 10 | def perf_native(n_rec, n_iter): 11 | start = time.time() 12 | res = fib_rec_py(n_rec) 13 | finish = time.time() 14 | print(f"[python] fib_rec_py({n_rec}) = {res}, time = {finish - start}") 15 | 16 | start = time.time() 17 | res = fib_iter_py(n_iter) 18 | finish = time.time() 19 | print(f"[python] fib_iter_py({n_iter}) = {res}, time = {finish - start}") 20 | print("------------ \n") 21 | 22 | 23 | def perf_ctypes(n_rec, n_iter): 24 | lib = ctypes.cdll.LoadLibrary("./fib_ctypes/libfibutils.so") 25 | 26 | lib.fib_rec_ctypes.argstype = [ctypes.c_int] 27 | lib.fib_rec_ctypes.restype = ctypes.c_int 28 | 29 | lib.fib_iter_ctypes.argstype = [ctypes.c_int] 30 | lib.fib_iter_ctypes.restype = ctypes.c_int 31 | 32 | start = time.time() 33 | res = lib.fib_rec_ctypes(n_rec) 34 | finish = time.time() 35 | print(f"[ctypes] fib_rec_ctypes({n_rec}) = {res}, time = {finish - start}") 36 | 37 | start = time.time() 38 | res = lib.fib_iter_ctypes(n_iter) 39 | finish = time.time() 40 | print(f"[ctypes] fib_iter_ctypes({n_iter}) = {res}, time = {finish - start}") 41 | print("------------ \n") 42 | 43 | 44 | def perf_cffi(n_rec, n_iter): 45 | ffi = cffi.FFI() 46 | lib = ffi.dlopen("./fib_ctypes/libfibutils.so") 47 | 48 | ffi.cdef( 49 | "int fib_rec_ctypes(int n);\n" 50 | "int fib_iter_ctypes(int n);\n" 51 | ) 52 | 53 | start = time.time() 54 | res = lib.fib_rec_ctypes(n_rec) 55 | finish = time.time() 56 | print(f"[cffi] fib_rec_ctypes({n_rec}) = {res}, time = {finish - start}") 57 | 58 | start = time.time() 59 | res = lib.fib_iter_ctypes(n_iter) 60 | finish = time.time() 61 | print(f"[cffi] fib_iter_ctypes({n_iter}) = {res}, time = {finish - start}") 62 | print("------------ \n") 63 | 64 | 65 | def perf_capi(n_rec, n_iter): 66 | start = time.time() 67 | res = fibutils_capi.fib_rec_c(n_rec) 68 | finish = time.time() 69 | print(f"[capi] fib_rec_c({n_rec}) = {res}, time = {finish - start}") 70 | 71 | start = time.time() 72 | res = fibutils_capi.fib_iter_c(n_iter) 73 | finish = time.time() 74 | print(f"[capi] fib_iter_c({n_iter}) = {res}, time = {finish - start}") 75 | print("------------ \n") 76 | 77 | 78 | def perf_cython(n_rec, n_iter): 79 | start = time.time() 80 | res = fibcyth.fib_rec_cy(n_rec) 81 | finish = time.time() 82 | print(f"[cython] fib_rec_cy({n_rec}) = {res}, time = {finish - start}") 83 | 84 | start = time.time() 85 | res = fibcyth.fib_iter_cy(n_iter) 86 | finish = time.time() 87 | print(f"[cython] fib_iter_cy({n_iter}) = {res}, time = {finish - start}") 88 | print("------------ \n") 89 | 90 | 91 | def run(): 92 | n_rec = 37 93 | n_iter = 45 94 | 95 | perf_native(n_rec, n_iter) 96 | perf_ctypes(n_rec, n_iter) 97 | perf_cffi(n_rec, n_iter) 98 | perf_capi(n_rec, n_iter) 99 | perf_cython(n_rec, n_iter) 100 | 101 | 102 | if __name__ == "__main__": 103 | run() 104 | -------------------------------------------------------------------------------- /lesson-11/check_hints.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Iterable, Sequence 2 | from dataclasses import dataclass 3 | from typing import ( 4 | Any, 5 | TYPE_CHECKING, 6 | NewType, 7 | Generator, 8 | TypeVar, 9 | runtime_checkable, 10 | Protocol, 11 | ) 12 | 13 | 14 | @dataclass 15 | class User: 16 | user_id: str 17 | name: str 18 | age: int 19 | 20 | def __bool__(self): 21 | return self.user_id in {"123", "a1w3"} 22 | 23 | 24 | @dataclass 25 | class LibraryAccount(User): 26 | pass 27 | 28 | 29 | type UserData = dict[str, str | int] 30 | type OptUserData = UserData | None 31 | 32 | 33 | def fetch_user(user_id: str) -> UserData: 34 | return { 35 | "user_id": user_id, 36 | "name": "Steve", 37 | "age": 99, 38 | } 39 | 40 | 41 | def get_user_data(user: User) -> OptUserData: 42 | if not user: 43 | return None 44 | 45 | data = fetch_user(user.user_id) 46 | 47 | return data 48 | 49 | 50 | def run_get_user_data() -> None: 51 | user = User("123", "Steve", 99) 52 | res: OptUserData = get_user_data(user) 53 | print(f"1: get_user_data={res}") 54 | 55 | user = LibraryAccount("a1w3", "Lib", 105) 56 | res = get_user_data(user) 57 | print(f"2: get_user_data={res}") 58 | 59 | user = User("not_a_user", "Steve", 99) 60 | res = get_user_data(user) 61 | print(f"3: get_user_data={res}") 62 | 63 | 64 | Celsius = NewType("Celsius", float) 65 | Fareng = NewType("Fareng", float) 66 | 67 | 68 | def convert_c_to_f(temp: Celsius) -> Fareng: 69 | return Fareng(temp * 9 / 5 + 32) 70 | 71 | 72 | def gen_celsius_temps(start: Celsius) -> Generator[Celsius, None, str]: 73 | yield start 74 | yield Celsius(start + 10) 75 | yield Celsius(start + 20) 76 | 77 | return "gen finished" 78 | 79 | 80 | @runtime_checkable 81 | class Comparable(Protocol): 82 | def __lt__(self: "T", other: "T") -> bool: ... 83 | def __gt__(self: "T", other: "T") -> bool: ... 84 | 85 | 86 | T = TypeVar("T", bound=Comparable) 87 | 88 | def get_max_temp(temps: Iterable[T]) -> T | None: 89 | if not temps: 90 | return None 91 | 92 | return max(temps) 93 | 94 | 95 | # def get_max_temp(temps: Iterable[Celsius]) -> Celsius | None: 96 | # if not temps: 97 | # return None 98 | # 99 | # return max(temps) 100 | 101 | 102 | def get_first_temp[V](temps: Sequence[V]) -> V | None: 103 | if not temps: 104 | return None 105 | 106 | return temps[0] 107 | 108 | 109 | def run_temperatures() -> None: 110 | temp_c = Celsius(36.6) 111 | temp_f = convert_c_to_f(temp_c) 112 | print(f"1: {temp_c=}, {temp_f=}, {type(temp_c)}") 113 | 114 | temp_c = Celsius(-40) 115 | temp_f = convert_c_to_f(temp_c) 116 | print(f"2: {temp_c=}, {temp_f=}, {type(temp_c)}") 117 | 118 | temps: Iterable[Celsius] = [Celsius(36.6), Celsius(-40)] 119 | max_temp = get_max_temp(temps) 120 | print(f"3: {max_temp=}, {temps=}") 121 | 122 | temps = gen_celsius_temps(Celsius(10)) 123 | max_temp = get_max_temp(temps) 124 | print(f"4: {max_temp=}, {temps=}") 125 | 126 | temps_seq: Sequence[Celsius] = [Celsius(36.6), Celsius(-40)] 127 | first_temp = get_first_temp(temps_seq) 128 | print(f"5: {first_temp=}, {temps_seq=}") 129 | 130 | temps_1: tuple[Celsius, ...] = (Celsius(36.6), Celsius(-40), Celsius(0)) 131 | first_temp = get_first_temp(temps_1) 132 | print(f"6: {first_temp=}, {temps_1=}") 133 | 134 | temps_int = [36, -40, 50] 135 | max_temp7 = get_max_temp(temps_int) 136 | print(f"7: {max_temp7=}, {temps_int=}") 137 | 138 | first_temp1: int | None = get_first_temp(temps_int) 139 | print(f"8: {first_temp1=}, {temps_int=}") 140 | 141 | 142 | @runtime_checkable 143 | class Predictable(Protocol): 144 | def predict(self) -> float: 145 | pass 146 | 147 | 148 | class RandomForest: 149 | def predict(self) -> float: 150 | return 0.95 151 | 152 | 153 | def calc_prediction(model: Predictable) -> float: 154 | #return model.predict() 155 | return 0.1 156 | 157 | 158 | def run_predictable() -> None: 159 | model = RandomForest() 160 | score = calc_prediction(model) 161 | print(f"{score=}") 162 | 163 | user = User("123", "Steve", 99) 164 | print(f"{isinstance(model, Predictable)=}, {isinstance(user, Predictable)=}") 165 | 166 | score = calc_prediction(user) 167 | print(f"{score=}") 168 | 169 | 170 | if __name__ == "__main__": 171 | print(issubclass(Iterable, Sequence), issubclass(Sequence, Iterable)) 172 | 173 | print(f"{TYPE_CHECKING=}") 174 | run_get_user_data() 175 | 176 | print() 177 | run_temperatures() 178 | 179 | print() 180 | run_predictable() 181 | 182 | print("ok") 183 | -------------------------------------------------------------------------------- /lesson-11/lesson-11.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mailcourses/deep_python_spring_2025/ede9fa206266b26d1da2c1857c8c41482677c3b9/lesson-11/lesson-11.pdf -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | flake8==7.1.0 2 | pylint==3.2.7 3 | pytest==8.3.2 4 | coverage==7.6.1 5 | --------------------------------------------------------------------------------