├── .coveragerc ├── .gitignore ├── .readthedocs.yml ├── .style.yapf ├── .travis.yml ├── CHEATSHEET.rst ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── LICENSE.APACHE2 ├── LICENSE.MIT ├── MANIFEST.in ├── README.rst ├── ci ├── rtd-requirements.in ├── rtd-requirements.txt └── travis.sh ├── docs ├── Makefile ├── make.bat └── source │ ├── _static │ └── .gitkeep │ ├── conf.py │ ├── history.rst │ └── index.rst ├── newsfragments ├── .gitkeep └── README.rst ├── pyproject.toml ├── pytest.ini ├── setup.py ├── test-requirements.in ├── test-requirements.txt └── trimeter ├── __init__.py ├── _impl.py ├── _tests ├── __init__.py └── test_basics.py └── _version.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch=True 3 | source=trimeter 4 | 5 | [report] 6 | precision = 1 7 | exclude_lines = 8 | pragma: no cover 9 | abc.abstractmethod 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Add any project-specific files here: 2 | 3 | 4 | # Sphinx docs 5 | docs/build/ 6 | 7 | # Byte-compiled / optimized / DLL files 8 | __pycache__/ 9 | *.py[cod] 10 | *~ 11 | \#* 12 | .#* 13 | 14 | # C extensions 15 | *.so 16 | 17 | # Distribution / packaging 18 | .Python 19 | /build/ 20 | /develop-eggs/ 21 | /dist/ 22 | /eggs/ 23 | /lib/ 24 | /lib64/ 25 | /parts/ 26 | /sdist/ 27 | /var/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | 32 | # Installer logs 33 | pip-log.txt 34 | 35 | # Unit test / coverage reports 36 | htmlcov/ 37 | .tox/ 38 | .coverage 39 | .coverage.* 40 | .cache 41 | .pytest_cache 42 | nosetests.xml 43 | coverage.xml 44 | 45 | # Translations 46 | *.mo 47 | 48 | # Mr Developer 49 | .mr.developer.cfg 50 | .project 51 | .pydevproject 52 | 53 | # Rope 54 | .ropeproject 55 | 56 | # Django stuff: 57 | *.log 58 | *.pot 59 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # https://docs.readthedocs.io/en/latest/yaml-config.html 2 | formats: 3 | - htmlzip 4 | - epub 5 | 6 | requirements_file: ci/rtd-requirements.txt 7 | 8 | # Currently RTD's default image only has 3.5 9 | # This gets us 3.6 (and hopefully 3.7 in the future) 10 | # https://docs.readthedocs.io/en/latest/yaml-config.html#build-image 11 | build: 12 | image: latest 13 | 14 | python: 15 | version: 3 16 | pip_install: True 17 | -------------------------------------------------------------------------------- /.style.yapf: -------------------------------------------------------------------------------- 1 | [style] 2 | # Align closing bracket with visual indentation. 3 | align_closing_bracket_with_visual_indent=True 4 | 5 | # Allow dictionary keys to exist on multiple lines. For example: 6 | # 7 | # x = { 8 | # ('this is the first element of a tuple', 9 | # 'this is the second element of a tuple'): 10 | # value, 11 | # } 12 | allow_multiline_dictionary_keys=False 13 | 14 | # Allow lambdas to be formatted on more than one line. 15 | allow_multiline_lambdas=False 16 | 17 | # Insert a blank line before a class-level docstring. 18 | blank_line_before_class_docstring=False 19 | 20 | # Insert a blank line before a 'def' or 'class' immediately nested 21 | # within another 'def' or 'class'. For example: 22 | # 23 | # class Foo: 24 | # # <------ this blank line 25 | # def method(): 26 | # ... 27 | blank_line_before_nested_class_or_def=False 28 | 29 | # Do not split consecutive brackets. Only relevant when 30 | # dedent_closing_brackets is set. For example: 31 | # 32 | # call_func_that_takes_a_dict( 33 | # { 34 | # 'key1': 'value1', 35 | # 'key2': 'value2', 36 | # } 37 | # ) 38 | # 39 | # would reformat to: 40 | # 41 | # call_func_that_takes_a_dict({ 42 | # 'key1': 'value1', 43 | # 'key2': 'value2', 44 | # }) 45 | coalesce_brackets=False 46 | 47 | # The column limit. 48 | column_limit=79 49 | 50 | # Indent width used for line continuations. 51 | continuation_indent_width=4 52 | 53 | # Put closing brackets on a separate line, dedented, if the bracketed 54 | # expression can't fit in a single line. Applies to all kinds of brackets, 55 | # including function definitions and calls. For example: 56 | # 57 | # config = { 58 | # 'key1': 'value1', 59 | # 'key2': 'value2', 60 | # } # <--- this bracket is dedented and on a separate line 61 | # 62 | # time_series = self.remote_client.query_entity_counters( 63 | # entity='dev3246.region1', 64 | # key='dns.query_latency_tcp', 65 | # transform=Transformation.AVERAGE(window=timedelta(seconds=60)), 66 | # start_ts=now()-timedelta(days=3), 67 | # end_ts=now(), 68 | # ) # <--- this bracket is dedented and on a separate line 69 | dedent_closing_brackets=True 70 | 71 | # Place each dictionary entry onto its own line. 72 | each_dict_entry_on_separate_line=True 73 | 74 | # The regex for an i18n comment. The presence of this comment stops 75 | # reformatting of that line, because the comments are required to be 76 | # next to the string they translate. 77 | i18n_comment= 78 | 79 | # The i18n function call names. The presence of this function stops 80 | # reformattting on that line, because the string it has cannot be moved 81 | # away from the i18n comment. 82 | i18n_function_call= 83 | 84 | # Indent the dictionary value if it cannot fit on the same line as the 85 | # dictionary key. For example: 86 | # 87 | # config = { 88 | # 'key1': 89 | # 'value1', 90 | # 'key2': value1 + 91 | # value2, 92 | # } 93 | indent_dictionary_value=True 94 | 95 | # The number of columns to use for indentation. 96 | indent_width=4 97 | 98 | # Join short lines into one line. E.g., single line 'if' statements. 99 | join_multiple_lines=False 100 | 101 | # Use spaces around default or named assigns. 102 | spaces_around_default_or_named_assign=False 103 | 104 | # Use spaces around the power operator. 105 | spaces_around_power_operator=False 106 | 107 | # The number of spaces required before a trailing comment. 108 | spaces_before_comment=2 109 | 110 | # Insert a space between the ending comma and closing bracket of a list, 111 | # etc. 112 | space_between_ending_comma_and_closing_bracket=False 113 | 114 | # Split before arguments if the argument list is terminated by a 115 | # comma. 116 | split_arguments_when_comma_terminated=True 117 | 118 | # Set to True to prefer splitting before '&', '|' or '^' rather than 119 | # after. 120 | split_before_bitwise_operator=True 121 | 122 | # Split before a dictionary or set generator (comp_for). For example, note 123 | # the split before the 'for': 124 | # 125 | # foo = { 126 | # variable: 'Hello world, have a nice day!' 127 | # for variable in bar if variable != 42 128 | # } 129 | split_before_dict_set_generator=True 130 | 131 | # If an argument / parameter list is going to be split, then split before 132 | # the first argument. 133 | split_before_first_argument=True 134 | 135 | # Set to True to prefer splitting before 'and' or 'or' rather than 136 | # after. 137 | split_before_logical_operator=True 138 | 139 | # Split named assignments onto individual lines. 140 | split_before_named_assigns=True 141 | 142 | # The penalty for splitting right after the opening bracket. 143 | split_penalty_after_opening_bracket=30 144 | 145 | # The penalty for splitting the line after a unary operator. 146 | split_penalty_after_unary_operator=10000 147 | 148 | # The penalty for splitting right before an if expression. 149 | split_penalty_before_if_expr=0 150 | 151 | # The penalty of splitting the line around the '&', '|', and '^' 152 | # operators. 153 | split_penalty_bitwise_operator=300 154 | 155 | # The penalty for characters over the column limit. 156 | split_penalty_excess_character=4500 157 | 158 | # The penalty incurred by adding a line split to the unwrapped line. The 159 | # more line splits added the higher the penalty. 160 | split_penalty_for_added_line_split=30 161 | 162 | # The penalty of splitting a list of "import as" names. For example: 163 | # 164 | # from a_very_long_or_indented_module_name_yada_yad import (long_argument_1, 165 | # long_argument_2, 166 | # long_argument_3) 167 | # 168 | # would reformat to something like: 169 | # 170 | # from a_very_long_or_indented_module_name_yada_yad import ( 171 | # long_argument_1, long_argument_2, long_argument_3) 172 | split_penalty_import_names=0 173 | 174 | # The penalty of splitting the line around the 'and' and 'or' 175 | # operators. 176 | split_penalty_logical_operator=0 177 | 178 | # Use the Tab character for indentation. 179 | use_tabs=False 180 | 181 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | dist: focal 3 | 4 | matrix: 5 | include: 6 | # These are quick and often catch errors, so list them first 7 | - python: 3.8 8 | env: CHECK_DOCS=1 9 | - python: 3.8 10 | env: CHECK_FORMATTING=1 11 | # The pypy tests are slow, so we list them first 12 | - python: pypy3.6-7.2.0 13 | dist: bionic 14 | - language: generic 15 | env: PYPY_NIGHTLY_BRANCH=py3.6 16 | # Uncomment if you want to test on pypy nightly: 17 | # - language: generic 18 | # env: USE_PYPY_NIGHTLY=1 19 | - python: 3.6-dev 20 | # As of 2018-07-05, Travis's 3.7 and 3.8 builds only work if you 21 | # use dist: xenial AND sudo: required 22 | # See: https://github.com/python-trio/trio/pull/556#issuecomment-402879391 23 | - python: 3.7-dev 24 | - python: 3.8-dev 25 | - python: 3.9-dev 26 | 27 | script: 28 | - ci/travis.sh 29 | -------------------------------------------------------------------------------- /CHEATSHEET.rst: -------------------------------------------------------------------------------- 1 | Tips 2 | ==== 3 | 4 | To run tests 5 | ------------ 6 | 7 | * Install requirements: ``pip install -r test-requirements.txt`` 8 | (possibly in a virtualenv) 9 | 10 | * Actually run the tests: ``pytest trimeter`` 11 | 12 | 13 | To run yapf 14 | ----------- 15 | 16 | * Show what changes yapf wants to make: ``yapf -rpd setup.py 17 | trimeter`` 18 | 19 | * Apply all changes directly to the source tree: ``yapf -rpi setup.py 20 | trimeter`` 21 | 22 | 23 | To make a release 24 | ----------------- 25 | 26 | * Update the version in ``trimeter/_version.py`` 27 | 28 | * Run ``towncrier`` to collect your release notes. 29 | 30 | * Review your release notes. 31 | 32 | * Check everything in. 33 | 34 | * Double-check it all works, docs build, etc. 35 | 36 | * Build your sdist and wheel: ``python setup.py sdist bdist_wheel`` 37 | 38 | * Upload to PyPI: ``twine upload dist/*`` 39 | 40 | * Use ``git tag`` to tag your version. 41 | 42 | * Don't forget to ``git push --tags``. 43 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | For the Trio code of conduct, see: 2 | https://trio.readthedocs.io/en/latest/code-of-conduct.html 3 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | For the Trio contributing guide, see: 2 | https://trio.readthedocs.io/en/latest/contributing.html 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This software is made available under the terms of *either* of the 2 | licenses found in LICENSE.APACHE2 or LICENSE.MIT. Contributions to are 3 | made under the terms of *both* these licenses. 4 | -------------------------------------------------------------------------------- /LICENSE.APACHE2: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /LICENSE.MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst CHEATSHEET.rst LICENSE* CODE_OF_CONDUCT* CONTRIBUTING* 2 | include .coveragerc .style.yapf 3 | include test-requirements.txt 4 | recursive-include docs * 5 | prune docs/build 6 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. image:: https://img.shields.io/badge/chat-join%20now-blue.svg 2 | :target: https://gitter.im/python-trio/general 3 | :alt: Join chatroom 4 | 5 | .. image:: https://img.shields.io/badge/docs-read%20now-blue.svg 6 | :target: https://trimeter.readthedocs.io/en/latest/?badge=latest 7 | :alt: Documentation Status 8 | 9 | .. image:: https://img.shields.io/pypi/v/trimeter.svg 10 | :target: https://pypi.org/project/trimeter 11 | :alt: Latest PyPi version 12 | 13 | .. image:: https://travis-ci.org/python-trio/trimeter.svg?branch=master 14 | :target: https://travis-ci.org/python-trio/trimeter 15 | :alt: Automated test status 16 | 17 | .. image:: https://codecov.io/gh/python-trio/trimeter/branch/master/graph/badge.svg 18 | :target: https://codecov.io/gh/python-trio/trimeter 19 | :alt: Test coverage 20 | 21 | Warning 22 | ======= 23 | 24 | This library isn't ready for release yet. Feedback welcome! 25 | 26 | 27 | Trimeter 28 | ======== 29 | 30 | Trio is a friendly Python library for async concurrency and 31 | networking. Trimeter is a simple but powerful job scheduler for 32 | programs using Trio, released under your choice of the MIT or Apache 2 33 | licenses. 34 | 35 | Trimeter's core purpose is to make it easy to execute lots tasks 36 | concurrently, with rich options to **control the degree of 37 | concurrency** and to **collect the task results**. 38 | 39 | Say you have 1000 urls that you want to fetch and process somehow: 40 | 41 | .. code-block:: python3 42 | 43 | # Old slow way 44 | for url in urls: 45 | await fetch_and_process(url) 46 | 47 | That's slow, so you want to do several at the same time... but to 48 | avoid overloading the network, you want to limit it to at most 5 calls 49 | at once. Oh, and there's a request quota, so we have to throttle it 50 | down to 1 per second. No problem: 51 | 52 | .. code-block:: python3 53 | 54 | # New and fancy way 55 | await trimeter.run_on_each( 56 | fetch_and_process, urls, max_at_once=5, max_per_second=1 57 | ) 58 | 59 | What if we don't know the whole list of urls up front? No worries, 60 | just pass in an async iterable instead, and Trimeter will do the right 61 | thing. 62 | 63 | What if we want to get the result from each call as it finishes, so we 64 | can do something further with it? Just use ``amap`` (= short for 65 | "async `map 66 | `__"): 67 | 68 | .. code-block:: python3 69 | 70 | async with trimeter.amap(fetch_and_process, urls, ...) as results: 71 | # Then iterate over the return values, as they become available 72 | # (i.e., not necessarily in the original order) 73 | async for result in results: 74 | ... 75 | 76 | Of course ``amap`` also accepts throttling options like 77 | ``max_at_once``, ``max_per_second``, etc. 78 | 79 | What if we want to use the `outcome library 80 | `__ to capture exceptions, so one 81 | call crashing doesn't terminate the whole program? And also, we want 82 | to pass through the original url alongside each result, so we know 83 | which result goes with which url? 84 | 85 | .. code-block:: python3 86 | 87 | async with trimeter.amap( 88 | fetch_and_process, 89 | urls, 90 | capture_outcome=True, 91 | include_value=True, 92 | ) as outcomes: 93 | # Then iterate over the return values, as they become available 94 | # (i.e., not necessarily in the original order) 95 | async for url, outcome in outcomes: 96 | try: 97 | return_value = outcome.unwrap() 98 | except Exception as exc: 99 | print(f"error while processing {url}: {exc!r}") 100 | 101 | What if we just want to call a few functions in parallel and then get 102 | the results as a list, like `asyncio.gather 103 | `__ 104 | or `Promise.all 105 | `__? 106 | 107 | .. code-block:: python3 108 | 109 | return_values = await trimeter.run_all([ 110 | async_fn1, 111 | async_fn2, 112 | functools.partial(async_fn3, extra_arg, kwarg="yeah"), 113 | ]) 114 | 115 | Of course, this takes all the same options as the other functions, so 116 | you can control the degree of parallelism, use ``capture_outcome`` to 117 | capture exceptions, and so forth. 118 | 119 | For more details, see `the fine manual 120 | `__. 121 | 122 | 123 | Can you summarize that in iambic trimeter? 124 | ------------------------------------------ 125 | 126 | `Iambic trimeter `__? 127 | No problem: 128 | 129 | | Trimeter gives you tools 130 | | for running lots of tasks 131 | | to do your work real fast 132 | | but not so fast you crash. 133 | 134 | 135 | Code of conduct 136 | --------------- 137 | 138 | Contributors are requested to follow our `code of conduct 139 | `__ in all 140 | project spaces. 141 | -------------------------------------------------------------------------------- /ci/rtd-requirements.in: -------------------------------------------------------------------------------- 1 | # RTD is currently installing 1.5.3, which has a bug in :lineno-match: 2 | sphinx >= 1.6.1 3 | sphinx_rtd_theme 4 | sphinxcontrib-trio 5 | -------------------------------------------------------------------------------- /ci/rtd-requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile --output-file=rtd-requirements.txt rtd-requirements.in 6 | # 7 | alabaster==0.7.12 # via sphinx 8 | babel==2.8.0 # via sphinx 9 | certifi==2020.6.20 # via requests 10 | chardet==3.0.4 # via requests 11 | docutils==0.16 # via sphinx 12 | idna==2.10 # via requests 13 | imagesize==1.2.0 # via sphinx 14 | jinja2==2.11.2 # via sphinx 15 | markupsafe==1.1.1 # via jinja2 16 | packaging==20.4 # via sphinx 17 | pygments==2.6.1 # via sphinx 18 | pyparsing==2.4.7 # via packaging 19 | pytz==2020.1 # via babel 20 | requests==2.24.0 # via sphinx 21 | six==1.15.0 # via packaging 22 | snowballstemmer==2.0.0 # via sphinx 23 | sphinx-rtd-theme==0.5.0 # via -r rtd-requirements.in 24 | sphinx==3.2.1 # via -r rtd-requirements.in, sphinx-rtd-theme, sphinxcontrib-trio 25 | sphinxcontrib-applehelp==1.0.2 # via sphinx 26 | sphinxcontrib-devhelp==1.0.2 # via sphinx 27 | sphinxcontrib-htmlhelp==1.0.3 # via sphinx 28 | sphinxcontrib-jsmath==1.0.1 # via sphinx 29 | sphinxcontrib-qthelp==1.0.3 # via sphinx 30 | sphinxcontrib-serializinghtml==1.1.4 # via sphinx 31 | sphinxcontrib-trio==1.1.2 # via -r rtd-requirements.in 32 | urllib3==1.25.10 # via requests 33 | 34 | # The following packages are considered to be unsafe in a requirements file: 35 | # setuptools 36 | -------------------------------------------------------------------------------- /ci/travis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | YAPF_VERSION=0.22.0 6 | 7 | if [ "$TRAVIS_OS_NAME" = "osx" ]; then 8 | curl -Lo macpython.pkg https://www.python.org/ftp/python/${MACPYTHON}/python-${MACPYTHON}-macosx10.6.pkg 9 | sudo installer -pkg macpython.pkg -target / 10 | ls /Library/Frameworks/Python.framework/Versions/*/bin/ 11 | PYTHON_EXE=/Library/Frameworks/Python.framework/Versions/*/bin/python3 12 | # The pip in older MacPython releases doesn't support a new enough TLS 13 | curl https://bootstrap.pypa.io/get-pip.py | sudo $PYTHON_EXE 14 | sudo $PYTHON_EXE -m pip install virtualenv 15 | $PYTHON_EXE -m virtualenv testenv 16 | source testenv/bin/activate 17 | fi 18 | 19 | if [ "$USE_PYPY_NIGHTLY" = "1" ]; then 20 | curl -fLo pypy.tar.bz2 http://buildbot.pypy.org/nightly/py3.5/pypy-c-jit-latest-linux64.tar.bz2 21 | if [ ! -s pypy.tar.bz2 ]; then 22 | # We know: 23 | # - curl succeeded (200 response code; -f means "exit with error if 24 | # server returns 4xx or 5xx") 25 | # - nonetheless, pypy.tar.bz2 does not exist, or contains no data 26 | # This isn't going to work, and the failure is not informative of 27 | # anything involving this package. 28 | ls -l 29 | echo "PyPy3 nightly build failed to download – something is wrong on their end." 30 | echo "Skipping testing against the nightly build for right now." 31 | exit 0 32 | fi 33 | tar xaf pypy.tar.bz2 34 | # something like "pypy-c-jit-89963-748aa3022295-linux64" 35 | PYPY_DIR=$(echo pypy-c-jit-*) 36 | PYTHON_EXE=$PYPY_DIR/bin/pypy3 37 | ($PYTHON_EXE -m ensurepip \ 38 | && $PYTHON_EXE -m pip install virtualenv \ 39 | && $PYTHON_EXE -m virtualenv testenv) \ 40 | || (echo "pypy nightly is broken; skipping tests"; exit 0) 41 | source testenv/bin/activate 42 | fi 43 | 44 | if [ "$USE_PYPY_RELEASE_VERSION" != "" ]; then 45 | curl -fLo pypy.tar.bz2 https://bitbucket.org/squeaky/portable-pypy/downloads/pypy3.5-${USE_PYPY_RELEASE_VERSION}-linux_x86_64-portable.tar.bz2 46 | tar xaf pypy.tar.bz2 47 | # something like "pypy3.5-5.7.1-beta-linux_x86_64-portable" 48 | PYPY_DIR=$(echo pypy3.5-*) 49 | PYTHON_EXE=$PYPY_DIR/bin/pypy3 50 | $PYTHON_EXE -m ensurepip 51 | $PYTHON_EXE -m pip install virtualenv 52 | $PYTHON_EXE -m virtualenv testenv 53 | source testenv/bin/activate 54 | fi 55 | 56 | pip install -U pip setuptools wheel pep517 57 | 58 | if [ "$CHECK_FORMATTING" = "1" ]; then 59 | pip install yapf==${YAPF_VERSION} 60 | if ! yapf -rpd setup.py trimeter; then 61 | cat <NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 20 | echo.installed, then set the SPHINXBUILD environment variable to point 21 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 22 | echo.may add the Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /docs/source/_static/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/python-trio/trimeter/85d50671c3c754d149c6211fa32fe81d5b07b79d/docs/source/_static/.gitkeep -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Documentation build configuration file, created by 5 | # sphinx-quickstart on Sat Jan 21 19:11:14 2017. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | # 20 | import os 21 | import sys 22 | # So autodoc can import our package 23 | sys.path.insert(0, os.path.abspath('../..')) 24 | 25 | # Warn about all references to unknown targets 26 | nitpicky = True 27 | # Except for these ones, which we expect to point to unknown targets: 28 | nitpick_ignore = [ 29 | # Format is ("sphinx reference type", "string"), e.g.: 30 | ("py:obj", "bytes-like"), 31 | ] 32 | 33 | # -- General configuration ------------------------------------------------ 34 | 35 | # If your documentation needs a minimal Sphinx version, state it here. 36 | # 37 | # needs_sphinx = '1.0' 38 | 39 | # Add any Sphinx extension module names here, as strings. They can be 40 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 41 | # ones. 42 | extensions = [ 43 | 'sphinx.ext.autodoc', 44 | 'sphinx.ext.intersphinx', 45 | 'sphinx.ext.coverage', 46 | 'sphinx.ext.napoleon', 47 | 'sphinxcontrib_trio', 48 | ] 49 | 50 | intersphinx_mapping = { 51 | "python": ('https://docs.python.org/3', None), 52 | "trio": ('https://trio.readthedocs.io/en/stable', None), 53 | } 54 | 55 | autodoc_member_order = "bysource" 56 | 57 | # Add any paths that contain templates here, relative to this directory. 58 | templates_path = [] 59 | 60 | # The suffix(es) of source filenames. 61 | # You can specify multiple suffix as a list of string: 62 | # 63 | # source_suffix = ['.rst', '.md'] 64 | source_suffix = '.rst' 65 | 66 | # The master toctree document. 67 | master_doc = 'index' 68 | 69 | # General information about the project. 70 | project = 'trimeter' 71 | copyright = 'The trimeter authors' 72 | author = 'The trimeter authors' 73 | 74 | # The version info for the project you're documenting, acts as replacement for 75 | # |version| and |release|, also used in various other places throughout the 76 | # built documents. 77 | # 78 | # The short X.Y version. 79 | import trimeter 80 | version = trimeter.__version__ 81 | # The full version, including alpha/beta/rc tags. 82 | release = version 83 | 84 | # The language for content autogenerated by Sphinx. Refer to documentation 85 | # for a list of supported languages. 86 | # 87 | # This is also used if you do content translation via gettext catalogs. 88 | # Usually you set "language" from the command line for these cases. 89 | language = None 90 | 91 | # List of patterns, relative to source directory, that match files and 92 | # directories to ignore when looking for source files. 93 | # This patterns also effect to html_static_path and html_extra_path 94 | exclude_patterns = [] 95 | 96 | # The name of the Pygments (syntax highlighting) style to use. 97 | pygments_style = 'sphinx' 98 | 99 | # The default language for :: blocks 100 | highlight_language = 'python3' 101 | 102 | # If true, `todo` and `todoList` produce output, else they produce nothing. 103 | todo_include_todos = False 104 | 105 | 106 | # -- Options for HTML output ---------------------------------------------- 107 | 108 | # The theme to use for HTML and HTML Help pages. See the documentation for 109 | # a list of builtin themes. 110 | # 111 | #html_theme = 'alabaster' 112 | 113 | # We have to set this ourselves, not only because it's useful for local 114 | # testing, but also because if we don't then RTD will throw away our 115 | # html_theme_options. 116 | import sphinx_rtd_theme 117 | html_theme = 'sphinx_rtd_theme' 118 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 119 | 120 | # Theme options are theme-specific and customize the look and feel of a theme 121 | # further. For a list of options available for each theme, see the 122 | # documentation. 123 | # 124 | html_theme_options = { 125 | # default is 2 126 | # show deeper nesting in the RTD theme's sidebar TOC 127 | # https://stackoverflow.com/questions/27669376/ 128 | # I'm not 100% sure this actually does anything with our current 129 | # versions/settings... 130 | "navigation_depth": 4, 131 | "logo_only": True, 132 | } 133 | 134 | # Add any paths that contain custom static files (such as style sheets) here, 135 | # relative to this directory. They are copied after the builtin static files, 136 | # so a file named "default.css" will overwrite the builtin "default.css". 137 | html_static_path = ['_static'] 138 | 139 | 140 | # -- Options for HTMLHelp output ------------------------------------------ 141 | 142 | # Output file base name for HTML help builder. 143 | htmlhelp_basename = 'trimeterdoc' 144 | 145 | 146 | # -- Options for LaTeX output --------------------------------------------- 147 | 148 | latex_elements = { 149 | # The paper size ('letterpaper' or 'a4paper'). 150 | # 151 | # 'papersize': 'letterpaper', 152 | 153 | # The font size ('10pt', '11pt' or '12pt'). 154 | # 155 | # 'pointsize': '10pt', 156 | 157 | # Additional stuff for the LaTeX preamble. 158 | # 159 | # 'preamble': '', 160 | 161 | # Latex figure (float) alignment 162 | # 163 | # 'figure_align': 'htbp', 164 | } 165 | 166 | # Grouping the document tree into LaTeX files. List of tuples 167 | # (source start file, target name, title, 168 | # author, documentclass [howto, manual, or own class]). 169 | latex_documents = [ 170 | (master_doc, 'trimeter.tex', 'Trio Documentation', 171 | author, 'manual'), 172 | ] 173 | 174 | 175 | # -- Options for manual page output --------------------------------------- 176 | 177 | # One entry per manual page. List of tuples 178 | # (source start file, name, description, authors, manual section). 179 | man_pages = [ 180 | (master_doc, 'trimeter', 'trimeter Documentation', 181 | [author], 1) 182 | ] 183 | 184 | 185 | # -- Options for Texinfo output ------------------------------------------- 186 | 187 | # Grouping the document tree into Texinfo files. List of tuples 188 | # (source start file, target name, title, author, 189 | # dir menu entry, description, category) 190 | texinfo_documents = [ 191 | (master_doc, 'trimeter', 'trimeter Documentation', 192 | author, 'trimeter', 'A simple but powerful job scheduler for Trio programs', 193 | 'Miscellaneous'), 194 | ] 195 | -------------------------------------------------------------------------------- /docs/source/history.rst: -------------------------------------------------------------------------------- 1 | Release history 2 | =============== 3 | 4 | .. currentmodule:: trimeter 5 | 6 | .. towncrier release notes start 7 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. documentation master file, created by 2 | sphinx-quickstart on Sat Jan 21 19:11:14 2017. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | 7 | =============================================================== 8 | trimeter: A simple but powerful job scheduler for Trio programs 9 | =============================================================== 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | history.rst 15 | 16 | ==================== 17 | Indices and tables 18 | ==================== 19 | 20 | * :ref:`genindex` 21 | * :ref:`modindex` 22 | * :ref:`search` 23 | * :ref:`glossary` 24 | -------------------------------------------------------------------------------- /newsfragments/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/python-trio/trimeter/85d50671c3c754d149c6211fa32fe81d5b07b79d/newsfragments/.gitkeep -------------------------------------------------------------------------------- /newsfragments/README.rst: -------------------------------------------------------------------------------- 1 | Adding newsfragments 2 | ==================== 3 | 4 | This directory collects "newsfragments": short files that each contain 5 | a snippet of ReST-formatted text that will be added to the next 6 | release notes. This should be a description of aspects of the change 7 | (if any) that are relevant to users. (This contrasts with your commit 8 | message and PR description, which are a description of the change as 9 | relevant to people working on the code itself.) 10 | 11 | Each file should be named like ``..rst``, where 12 | ```` is an issue numbers, and ```` is one of: 13 | 14 | * ``feature`` 15 | * ``bugfix`` 16 | * ``doc`` 17 | * ``removal`` 18 | * ``misc`` 19 | 20 | So for example: ``123.feature.rst``, ``456.bugfix.rst`` 21 | 22 | If your PR fixes an issue, use that number here. If there is no issue, 23 | then after you submit the PR and get the PR number you can add a 24 | newsfragment using that instead. 25 | 26 | Note that the ``towncrier`` tool will automatically 27 | reflow your text, so don't try to do any fancy formatting. You can 28 | install ``towncrier`` and then run ``towncrier --draft`` if you want 29 | to get a preview of how your change will look in the final release 30 | notes. 31 | 32 | 33 | Making releases 34 | =============== 35 | 36 | ``pip install towncrier``, then run ``towncrier``. (You can use 37 | ``towncrier --draft`` to get a preview of what this will do.) 38 | 39 | You can configure ``towncrier`` (for example: customizing the 40 | different types of changes) by modifying ``pyproject.toml``. 41 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.towncrier] 2 | package = "trimeter" 3 | filename = "docs/source/history.rst" 4 | directory = "newsfragments" 5 | underlines = ["-", "~", "^"] 6 | issue_format = "`#{issue} `__" 7 | 8 | [build-system] 9 | requires = [ 10 | "setuptools >= 50.0.3", 11 | "wheel >= 0.35.1", 12 | ] 13 | build-backend = "setuptools.build_meta" 14 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | trio_mode = true 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | exec(open("trimeter/_version.py", encoding="utf-8").read()) 4 | 5 | LONG_DESC = open("README.rst", encoding="utf-8").read() 6 | 7 | setup( 8 | name="trimeter", 9 | version=__version__, 10 | description="A simple but powerful job scheduler for Trio programs", 11 | url="https://github.com/python-trio/trimeter", 12 | long_description=LONG_DESC, 13 | long_description_content_type="text/x-rst", 14 | author="Nathaniel J. Smith", 15 | author_email="njs@pobox.com", 16 | license="MIT -or- Apache License 2.0", 17 | packages=find_packages(), 18 | install_requires=[ 19 | "trio >= 0.9.0", # For channels 20 | "outcome", 21 | "async_generator", 22 | "attrs", 23 | ], 24 | keywords=[ 25 | "async", 26 | "scheduler", 27 | "trio", 28 | ], 29 | python_requires=">=3.6", 30 | classifiers=[ 31 | "License :: OSI Approved :: MIT License", 32 | "License :: OSI Approved :: Apache Software License", 33 | "Framework :: Trio", 34 | "Operating System :: POSIX :: Linux", 35 | "Operating System :: MacOS :: MacOS X", 36 | "Operating System :: Microsoft :: Windows", 37 | "Programming Language :: Python :: 3 :: Only", 38 | "Programming Language :: Python :: Implementation :: CPython", 39 | "Programming Language :: Python :: Implementation :: PyPy", 40 | "Intended Audience :: Developers", 41 | ], 42 | ) 43 | -------------------------------------------------------------------------------- /test-requirements.in: -------------------------------------------------------------------------------- 1 | pytest 2 | pytest-cov 3 | pytest-trio 4 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile --output-file=test-requirements.txt test-requirements.in 6 | # 7 | async-generator==1.10 # via pytest-trio, trio 8 | attrs==20.1.0 # via outcome, pytest, trio 9 | coverage==5.2.1 # via pytest-cov 10 | idna==2.10 # via trio 11 | importlib-metadata==1.7.0 # via pluggy, pytest 12 | iniconfig==1.0.1 # via pytest 13 | more-itertools==8.5.0 # via pytest 14 | outcome==1.0.1 # via pytest-trio, trio 15 | packaging==20.4 # via pytest 16 | pluggy==0.13.1 # via pytest 17 | py==1.9.0 # via pytest 18 | pyparsing==2.4.7 # via packaging 19 | pytest-cov==2.10.1 # via -r test-requirements.in 20 | pytest-trio==0.6.0 # via -r test-requirements.in 21 | pytest==6.0.1 # via -r test-requirements.in, pytest-cov, pytest-trio 22 | six==1.15.0 # via packaging 23 | sniffio==1.1.0 # via trio 24 | sortedcontainers==2.2.2 # via trio 25 | toml==0.10.1 # via pytest 26 | trio==0.16.0 # via pytest-trio 27 | zipp==3.1.0 # via importlib-metadata 28 | -------------------------------------------------------------------------------- /trimeter/__init__.py: -------------------------------------------------------------------------------- 1 | """Top-level package for trimeter.""" 2 | 3 | from ._version import __version__ 4 | from ._impl import run_on_each, amap, run_all 5 | -------------------------------------------------------------------------------- /trimeter/_impl.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | import operator 3 | import abc 4 | 5 | import trio 6 | import outcome 7 | from async_generator import async_generator, yield_, asynccontextmanager 8 | import attr 9 | 10 | 11 | @async_generator 12 | async def iter2aiter(iterable): 13 | for value in iterable: 14 | await yield_(value) 15 | 16 | 17 | class Meter(abc.ABC): 18 | @abc.abstractmethod 19 | def new_state(self): 20 | pass 21 | 22 | 23 | class MeterState(abc.ABC): 24 | @abc.abstractmethod 25 | async def wait_task_can_start(self): 26 | pass 27 | 28 | @abc.abstractmethod 29 | def notify_task_started(self): 30 | pass 31 | 32 | @abc.abstractmethod 33 | def notify_task_finished(self): 34 | pass 35 | 36 | 37 | def _check_positive(self, attribute, value): 38 | if not value > 0: 39 | raise ValueError("{} must be > 0".format(attribute.name)) 40 | 41 | 42 | @attr.s(frozen=True) 43 | class MaxMeter: 44 | max_at_once = attr.ib(converter=operator.index, validator=_check_positive) 45 | 46 | def new_state(self): 47 | return MaxState(self) 48 | 49 | 50 | # XX if we ever get deadlock tracking it might be nice to use a 51 | # CapacityLimiter here and attribute the token ownership correctly 52 | # maybe notify_task_started should get the task object? Or run inside the 53 | # task? that would complicate startup slightly (can't call the next 54 | # wait_task_can_start until after calling notify_task_started), but nothing 55 | # nursery.start can't solve. 56 | class MaxState: 57 | def __init__(self, max_meter): 58 | self.sem = trio.Semaphore( 59 | initial_value=max_meter.max_at_once, 60 | max_value=max_meter.max_at_once 61 | ) 62 | 63 | async def wait_task_can_start(self): 64 | await self.sem.acquire() 65 | 66 | def notify_task_started(self): 67 | # Already acquired the semaphore in wait_task_can_start 68 | pass 69 | 70 | def notify_task_finished(self): 71 | self.sem.release() 72 | 73 | 74 | @attr.s(frozen=True) 75 | class TokenBucketMeter: 76 | max_per_second = attr.ib(converter=float, validator=_check_positive) 77 | max_burst = attr.ib( 78 | default=1, converter=operator.index, validator=_check_positive 79 | ) 80 | 81 | def __attrs_post_init__(self): 82 | _check_positive(self, "max_per_second", self.max_per_second) 83 | _check_positive(self, "max_burst", self.max_burst) 84 | 85 | def new_state(self): 86 | return TokenBucketState(self) 87 | 88 | 89 | class TokenBucketState: 90 | def __init__(self, token_bucket_meter): 91 | self._max_per_second = token_bucket_meter.max_per_second 92 | self._max_burst = token_bucket_meter.max_burst 93 | # In some cases it may make more sense to initialize to max_burst...? 94 | # We allow accumulating partial tokens, so this can be a float 95 | self._tokens = 1 96 | # maybe the start time should be passed in? that would let a keyed 97 | # token bucket meter 98 | self._last_update_time = trio.current_time() 99 | 100 | def _update(self): 101 | now = trio.current_time() 102 | elapsed = now - self._last_update_time 103 | self._tokens += elapsed * self._max_per_second 104 | self._tokens = max(self._tokens, self._max_burst) 105 | self._last_update_time = now 106 | 107 | async def wait_task_can_start(self): 108 | while True: 109 | self._update() 110 | if self._tokens >= 1: 111 | break 112 | next_token_after = (1 - self._tokens) / self._max_per_second 113 | await trio.sleep(next_token_after) 114 | 115 | def notify_task_started(self): 116 | # Have to make sure max_burst clipping is up to date before we take 117 | # our token 118 | self._update() 119 | assert self._tokens >= 1 120 | self._tokens -= 1 121 | 122 | def notify_task_finished(self): 123 | pass 124 | 125 | 126 | # XX should we have a special-case to allow KeyboardInterrupt to pass through? 127 | async def _worker(async_fn, value, index, config): 128 | if config.capture_outcome: 129 | result = await outcome.acapture(async_fn, value) 130 | else: 131 | result = await async_fn(value) 132 | if config.send_to is not None: 133 | if config.include_index and not config.include_value: 134 | result = (index, result) 135 | elif not config.include_index and config.include_value: 136 | result = (value, result) 137 | elif config.include_index and config.include_value: 138 | result = (index, value, result) 139 | await config.send_to.send(result) 140 | for meter_state in config.meter_states: 141 | meter_state.notify_task_finished() 142 | 143 | 144 | @attr.s(frozen=True) 145 | class Config: 146 | capture_outcome = attr.ib(converter=bool) 147 | include_index = attr.ib(converter=bool) 148 | include_value = attr.ib(converter=bool) 149 | send_to = attr.ib() 150 | meter_states = attr.ib() 151 | 152 | 153 | async def run_on_each( 154 | async_fn, 155 | iterable, 156 | *, 157 | max_at_once=None, 158 | max_per_second=None, 159 | max_burst=1, 160 | iterable_is_async="guess", 161 | send_to=None, 162 | capture_outcome=False, 163 | include_index=False, 164 | include_value=False, 165 | ): 166 | try: 167 | # XX: allow users to pass in their own custom meters 168 | meters = [] 169 | if max_at_once is not None: 170 | meters.append(MaxMeter(max_at_once)) 171 | if max_per_second is not None: 172 | meters.append(TokenBucketMeter(max_per_second, max_burst)) 173 | meter_states = [meter.new_state() for meter in meters] 174 | 175 | if iterable_is_async not in [True, False, "guess"]: 176 | raise ValueError("iterable_is_async must be bool or 'guess'") 177 | if iterable_is_async == "guess": 178 | iterable_is_async = hasattr(iterable, "__aiter__") 179 | if not iterable_is_async: 180 | iterable = iter2aiter(iterable) 181 | 182 | config = Config( 183 | capture_outcome=capture_outcome, 184 | include_index=include_index, 185 | include_value=include_value, 186 | send_to=send_to, 187 | meter_states=meter_states, 188 | ) 189 | 190 | if config.capture_outcome and config.send_to is None: 191 | raise ValueError("if capture_outcome=True, send_to cannot be None") 192 | 193 | async with trio.open_nursery() as nursery: 194 | index = 0 195 | async for value in iterable: 196 | for meter_state in meter_states: 197 | await meter_state.wait_task_can_start() 198 | for meter_state in meter_states: 199 | meter_state.notify_task_started() 200 | nursery.start_soon(_worker, async_fn, value, index, config) 201 | index += 1 202 | finally: 203 | if send_to is not None: 204 | await send_to.aclose() 205 | 206 | 207 | @asynccontextmanager 208 | @async_generator 209 | async def amap( 210 | async_fn, 211 | iterable, 212 | *, 213 | max_at_once=None, 214 | max_per_second=None, 215 | max_burst=1, 216 | iterable_is_async="guess", 217 | capture_outcome=False, 218 | include_index=False, 219 | include_value=False, 220 | max_buffer_size=0 221 | ): 222 | send_channel, receive_channel = trio.open_memory_channel(max_buffer_size) 223 | async with receive_channel: 224 | async with trio.open_nursery() as nursery: 225 | nursery.start_soon( 226 | partial( 227 | run_on_each, 228 | # Pass through: 229 | async_fn, 230 | iterable, 231 | max_at_once=max_at_once, 232 | max_per_second=max_per_second, 233 | max_burst=max_burst, 234 | iterable_is_async=iterable_is_async, 235 | capture_outcome=capture_outcome, 236 | include_index=include_index, 237 | include_value=include_value, 238 | # Not a simple pass-through: 239 | send_to=send_channel, 240 | ) 241 | ) 242 | await yield_(receive_channel) 243 | 244 | 245 | async def run_all( 246 | async_fns, 247 | *, 248 | max_at_once=None, 249 | max_per_second=None, 250 | max_burst=1, 251 | iterable_is_async="guess", 252 | capture_outcome=False, 253 | ): 254 | results = [None] * operator.length_hint(async_fns) 255 | results_len = 0 256 | async with amap( 257 | lambda fn: fn(), 258 | async_fns, 259 | max_at_once=max_at_once, 260 | max_per_second=max_per_second, 261 | iterable_is_async=iterable_is_async, 262 | capture_outcome=capture_outcome, 263 | include_index=True, 264 | ) as enum_results: 265 | async for index, result in enum_results: 266 | required_len = index + 1 267 | results_len = max(required_len, results_len) 268 | if required_len > len(results): 269 | results += [None] * (required_len - len(results)) 270 | results[index] = result 271 | del results[results_len:] 272 | return results 273 | -------------------------------------------------------------------------------- /trimeter/_tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/python-trio/trimeter/85d50671c3c754d149c6211fa32fe81d5b07b79d/trimeter/_tests/__init__.py -------------------------------------------------------------------------------- /trimeter/_tests/test_basics.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | import trio 3 | from trimeter import run_on_each, amap, run_all 4 | 5 | 6 | # Just the most basic smoke test of the three functions 7 | async def test_basics(): 8 | ran_on = [] 9 | 10 | async def afn(value): 11 | ran_on.append(value) 12 | return value + 1 13 | 14 | await run_on_each(afn, [1, 2, 3]) 15 | 16 | assert sorted(ran_on) == [1, 2, 3] 17 | 18 | async with amap(afn, [1, 2, 3]) as result_channel: 19 | results = [] 20 | async for result in result_channel: 21 | results.append(result) 22 | assert sorted(results) == [2, 3, 4] 23 | 24 | results = await run_all( 25 | [ 26 | partial(afn, 10), 27 | partial(afn, 11), 28 | partial(afn, 12), 29 | ] 30 | ) 31 | assert results == [11, 12, 13] 32 | -------------------------------------------------------------------------------- /trimeter/_version.py: -------------------------------------------------------------------------------- 1 | # This file is imported from __init__.py and exec'd from setup.py 2 | 3 | __version__ = "0.0.0" 4 | --------------------------------------------------------------------------------