├── .coveragerc ├── .github └── workflows │ └── test.yml ├── .gitignore ├── CONTRIBUTING.md ├── HISTORY.md ├── LICENSE ├── README.md ├── metafunctions.wpr ├── metafunctions ├── __init__.py ├── api.py ├── core │ ├── __init__.py │ ├── base.py │ ├── call_state.py │ ├── concurrent.py │ └── decorators.py ├── exceptions.py ├── map.py ├── operators.py ├── tests │ ├── __init__.py │ ├── simple_nodes.py │ ├── test_api.py │ ├── test_at_operator.py │ ├── test_call_state.py │ ├── test_concurrent.py │ ├── test_deferred_value.py │ ├── test_function_chain.py │ ├── test_function_merge.py │ ├── test_imports.py │ ├── test_map.py │ ├── test_pipe.py │ ├── test_simple_function.py │ ├── test_star.py │ ├── test_util.py │ └── util.py └── util.py ├── requirements-dev.txt ├── setup.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | # .coveragerc to control coverage.py 2 | [run] 3 | branch = True 4 | omit = */tests/* 5 | parallel = True -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Test 5 | 6 | on: 7 | push: 8 | branches: [ master, dev ] 9 | pull_request: 10 | 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ${{ matrix.os }} 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12.0-alpha - 3.12"] 20 | os: ["ubuntu-latest", "windows-latest", "macos-latest"] 21 | 22 | steps: 23 | - uses: actions/checkout@v2 24 | - name: Set up Python ${{ matrix.python-version }} 25 | uses: actions/setup-python@v2 26 | with: 27 | python-version: ${{ matrix.python-version }} 28 | - name: Install dependencies 29 | run: | 30 | python -m pip install --upgrade pip 31 | python -m pip install flake8 32 | pip install tox tox-gh-actions 33 | pip install -r requirements-dev.txt 34 | - name: Lint with flake8 35 | run: | 36 | # stop the build if there are Python syntax errors or undefined names 37 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 38 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 39 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 40 | - name: Run tox with tox-gh-actions 41 | run: tox -e py 42 | 43 | - name: Upload coverage 44 | uses: codecov/codecov-action@v2 45 | with: 46 | files: ./coverage.xml # optional 47 | fail_ci_if_error: true # optional (default = false) 48 | verbose: true # optional (default = false) 49 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *,cover 45 | 46 | # Translations 47 | *.mo 48 | *.pot 49 | 50 | # Django stuff: 51 | *.log 52 | *.sqlite3 53 | *.db 54 | 55 | # Sphinx documentation 56 | docs/_build/ 57 | 58 | # PyBuilder 59 | target/ 60 | 61 | #Wing 62 | *.wpu 63 | 64 | #osx 65 | .DS_Store 66 | 67 | # Jig 68 | .jig 69 | 70 | secrets.py 71 | *.json 72 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | To develop metafunctions, first create a virtual environment, then install the project in development mode: 2 | 3 | $ python3 -m venv venv 4 | $ . venv/bin/activate 5 | $ pip install -e . 6 | 7 | Then install extra dev requirements: 8 | 9 | $ pip install -r requirements-dev.txt 10 | 11 | The project uses Tox to run tests against multiple python versions. To run the tests: 12 | 13 | $ tox 14 | -------------------------------------------------------------------------------- /HISTORY.md: -------------------------------------------------------------------------------- 1 | # History 2 | 3 | ## 0.1.0 4 | * First release -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Tom Rutherford 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MetaFunctions 2 | ![GithubActions Badge](https://github.com/ForeverWintr/metafunctions/actions/workflows/test.yml/badge.svg) 3 | [![Codecov](https://codecov.io/gh/ForeverWintr/metafunctions/coverage.svg?branch=master)](https://codecov.io/gh/ForeverWintr/metafunctions) 4 | 5 | 6 | 7 | ## Metafunctions is a function composition and data pipelining library. 8 | It allows for data pipeline creation separate from execution, so instead of writing: 9 | 10 | ```python 11 | result = step3(step2(step1(data))) 12 | 13 | #or 14 | result_1 = step1(data) 15 | result_2 = step2(result_1) 16 | result_3 = step3(result_2) 17 | ``` 18 | 19 | You can write: 20 | 21 | ```python 22 | pipeline = step1 | step2 | step3 23 | result = pipeline(data) 24 | ``` 25 | 26 | ### Why do you need new syntax for pipelining functions? 27 | Well you may not *need* a new syntax, but the ability to compose a data pipeline before executing it does impart some advantages, such as: 28 | 29 | * **Reuse**. Compose a pipeline once and use it in multiple places, including as part of larger pipelines: 30 | ```python 31 | # load, parse, clean, validate, and format are functions 32 | preprocess = load | parse | clean | validate | format 33 | 34 | # preprocess is now a MetaFunction, and can be reused 35 | clean_data1 = preprocess('path/to/data/file') 36 | clean_data2 = preprocess('path/to/different/file') 37 | 38 | # Preprocess can be included in larger pipelines 39 | pipeline = preprocess | step1 | step2 | step3 40 | ``` 41 | * **Readability**. `step1 | step2 | step3` is both read and executed from left to right, unlike `step3(step2(step1()))`, which is executed from innermost function outwards. 42 | * **Inspection**. Can't remember what your MetaFunction does? `str` will tell you: 43 | ```python 44 | >>> str(preprocess) 45 | "(load | parse | clean | validate | format)" 46 | * **Advanced Composition**. Anything beyond simple function chaining becomes difficult using traditional methods. What if you want to send the result of `step1` to both steps `2` and `3`, then sum the results? The traditional approach requires an intermediate variable and can quickly become unwieldy: 47 | ```python 48 | result1 = step1(data) 49 | result2 = step2(result1) + step3(result1) 50 | ``` 51 | Using metafunctions, you can declare a pipeline that does the same thing: 52 | ```python 53 | pipeline = step1 | step2 + step3 54 | result = pipeline(data) 55 | ``` 56 | 57 | ## Installation 58 | 59 | MetaFunctions supports python 3.5+ (tested to 3.10+) 60 | 61 | `pip install metafunctions` 62 | 63 | ## How does it work? 64 | 65 | Conceptually, a MetaFunction is a function that contains other functions. When you call a MetaFunction, the MetaFunction calls the functions it contains. 66 | 67 | You can create a MetaFunction using the `node` decorator: 68 | ```python 69 | from metafunctions import node 70 | 71 | @node 72 | def get_name(prompt): 73 | return input(prompt) 74 | 75 | @node 76 | def say_hello(name): 77 | return 'Hello {}!'.format(name) 78 | ``` 79 | 80 | MetaFunctions override certain operators to allow for composition. For example, the following creates a new MetaFunction that combines `get_name` and `say_hello`: 81 | ```python 82 | greet = get_name | say_hello 83 | ``` 84 | 85 | When we call the `greet` MetaFunction, it calls both its internal functions in turn. 86 | ```python 87 | # First, `get_name` is called, which prints our prompt to the screen. 88 | # If we enter 'Tom' at the prompt, the second function returns the string 'Hello Tom!' 89 | greeting = greet('Please enter your name ') 90 | print(greeting) # Hello Tom! 91 | ``` 92 | 93 | MetaFunctions are also capable of upgrading regular functions to MetaFunctions at composition time, so we can simplify our example by composing `say_hello` directly with the builtin `input` and `print` functions: 94 | ```python 95 | >>> greet = input | say_hello | print 96 | >>> greet('Please enter your name: ') 97 | # Please enter your name: Tom 98 | # Hello Tom! 99 | ``` 100 | 101 | ## Features 102 | 103 | ### Helpful Tracebacks 104 | 105 | Errors in composed functions can be confusing. If an exception occurs in a MetaFunction, the exception traceback will tell you which function the exception occurred in. But what if that function appears multiple times in the data pipeline? 106 | 107 | Imagine this function, which downloads stringified numeric data from a web api: 108 | 109 | ```python 110 | >>> compute_value = (query_volume | float) * (query_price | float) 111 | >>> compute_value('http://prices.com/123') 112 | ``` 113 | 114 | Here we've assumed that `query_volume` and `query_price` will return strings that convert cleanly to floats, but what if something goes wrong? 115 | 116 | ``` 117 | >>> compute_value('http://prices.com/123') 118 | Traceback (most recent call last): 119 | File "", line 1, in 120 | ... 121 | builtins.ValueError: could not convert string to float: '$800' 122 | ``` 123 | 124 | We can deduce that float conversion failed, but *which* float function raised the exception? MetaFunctions addresses this by adding the `locate_error` metafunction decorator, which adds a location information string to any exception raised within the pipeline: 125 | 126 | ``` 127 | >>> from metafunctions import locate_error 128 | >>> with_location = locate_error(compute_value) 129 | >>> with_location('http://prices.com/123') 130 | Traceback (most recent call last): 131 | File "", line 1, in 132 | ... 133 | builtins.ValueError: could not convert string to float: '$800' 134 | Occured in the following function: ((query_volume | float) + (query_price | ->float<-)) 135 | ``` 136 | 137 | ### Advanced Pipeline Construction Tools 138 | 139 | Metafunctions provides utilities for constructing advanced function pipelines. 140 | 141 | * **store** / **recall**: store the output of the preceding function, and recall it later to pass to a different function. For example: 142 | 143 | ```python 144 | # The following pipeline sends the output of `a` to `b`, and also adds it to the output of `c` 145 | p = a | store('a') | b | recall('a') + c 146 | ``` 147 | 148 | * **mmap**: A MetaFunction decorator that wraps a function (or MetaFunction) and calls it once per item in the input it receives. This allows you to create loops in function pipelines: 149 | 150 | ```python 151 | (1, 2, 3) | mmap(process) # <- equivalent to (1, 2, 3) | (process & process & process) 152 | ``` 153 | `mmap` duplicates the behaviour of the builtin [`map`](https://docs.python.org/3/library/functions.html#map) function. 154 | 155 | * **star**: Calls the wrapped MetaFunction with *args instead of args (It's analogous to `lambda args, **kwargs: metafunction(*args, **kwargs)`). This allows you to incorporate functions that accept more than one parameter into your function pipeline: 156 | 157 | ```python 158 | @node 159 | def f(result1, result2): 160 | ... 161 | 162 | # When cmp is called, f will receive the results of both a and b as positional args 163 | cmp = (a & b) | star(f) 164 | ``` 165 | `star` can be combined with the above `mmap` to duplicate the behaviour of [`itertools.starmap`](https://docs.python.org/3/library/itertools.html#itertools.starmap): 166 | 167 | ```python 168 | starmap = star(mmap(f)) 169 | ``` 170 | 171 | For more discussion of `star`, see [this pull request](https://github.com/ForeverWintr/metafunctions/pull/9) 172 | 173 | * **concurrent**: 174 | *experimental, requires an os that provides `os.fork()`* 175 | 176 | Consider the following long running MetaFunction: 177 | 178 | ```python 179 | process_companies = get_company_data | filter | process 180 | process_customers = get_customer_data | filter | process 181 | 182 | process_all = process_companies & process_customers 183 | ``` 184 | 185 | Assuming the component functions in the `process_all` MetaFunction follow good functional practices and do not have side effects, it's easy to see that `process_companies` and `process_customers` are independent of each other. If that's the case, we can safely execute them in parallel. The `concurrent` metafunction decorator allows you to specify steps in the function pipeline to execute in parallel: 186 | 187 | ```python 188 | from metafunctions import concurrent 189 | 190 | do_large_calculation_async = concurrent(process_companies + process_customers) 191 | ``` 192 | 193 | `concurrent` can be combined with `mmap` to create an asynchronous map, similar to [`multiprocessing.pool.map`](https://docs.python.org/3/library/multiprocessing.html#multiprocessing.pool.Pool.map): 194 | 195 | ```python 196 | map_async = concurrent(mmap(f)) 197 | ``` 198 | -------------------------------------------------------------------------------- /metafunctions.wpr: -------------------------------------------------------------------------------- 1 | #!wing 2 | #!version=9.0 3 | ################################################################## 4 | # Wing project file # 5 | ################################################################## 6 | [project attributes] 7 | proj.directory-list = [{'dirloc': loc('metafunctions'), 8 | 'excludes': ['metafunctions.egg-info'], 9 | 'filter': '*', 10 | 'include_hidden': False, 11 | 'recursive': True, 12 | 'watch_for_changes': True}] 13 | proj.file-list = [loc('appveyor.yml'), 14 | loc('setup.py'), 15 | loc('tox.ini')] 16 | proj.file-type = 'shared' 17 | proj.launch-config = {loc('setup.py'): ('project', 18 | ('sdist\n', 19 | ''))} 20 | testing.auto-test-file-specs = (('glob', 21 | 'test_*.py'),) 22 | -------------------------------------------------------------------------------- /metafunctions/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "1.1.1" 2 | 3 | from metafunctions.api import ( 4 | node, 5 | bind_call_state, 6 | star, 7 | store, 8 | recall, 9 | concurrent, 10 | mmap, 11 | locate_error, 12 | ) 13 | -------------------------------------------------------------------------------- /metafunctions/api.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utility functions for use in function pipelines. 3 | """ 4 | import functools 5 | import typing as tp 6 | 7 | from metafunctions.core import MetaFunction 8 | from metafunctions.core import SimpleFunction 9 | from metafunctions.core import FunctionMerge 10 | from metafunctions.core import CallState 11 | from metafunctions import util 12 | from metafunctions.core.concurrent import ConcurrentMerge 13 | from metafunctions.map import MergeMap 14 | from metafunctions import operators 15 | 16 | 17 | def node(_func=None, *, name=None): 18 | """Turn the decorated function into a MetaFunction. 19 | 20 | Args: 21 | _func: Internal use. This will be the decorated function if node is used as a decorator 22 | with no params. 23 | 24 | Usage: 25 | 26 | @node 27 | def f(x): 28 | 29 | """ 30 | 31 | def decorator(function): 32 | newfunc = SimpleFunction(function, name=name) 33 | return newfunc 34 | 35 | if not _func: 36 | return decorator 37 | return decorator(_func) 38 | 39 | 40 | def bind_call_state(func): 41 | @functools.wraps(func) 42 | def provides_call_state(*args, **kwargs): 43 | call_state = kwargs.pop("call_state") 44 | return func(call_state, *args, **kwargs) 45 | 46 | provides_call_state._receives_call_state = True 47 | return provides_call_state 48 | 49 | 50 | def star(meta_function: MetaFunction) -> MetaFunction: 51 | """ 52 | star calls its Metafunction with *x instead of x. 53 | """ 54 | fname = str(meta_function) 55 | # This convoluted inline `if` just decides whether we should add brackets or not. 56 | @node( 57 | name="star{}".format(fname) 58 | if fname.startswith("(") 59 | else "star({})".format(fname) 60 | ) 61 | @functools.wraps(meta_function) 62 | def wrapper(args, **kwargs): 63 | return meta_function(*args, **kwargs) 64 | 65 | return wrapper 66 | 67 | 68 | def store(key): 69 | """Store the received output in the meta data dictionary under the given key.""" 70 | 71 | @node(name="store('{}')".format(key)) 72 | @bind_call_state 73 | def storer(call_state, val): 74 | call_state.data[key] = val 75 | return val 76 | 77 | return storer 78 | 79 | 80 | def recall(key, from_call_state: CallState = None): 81 | """Retrieve the given key from the meta data dictionary. Optionally, use `from_call_state` to 82 | specify a different call_state than the current one. 83 | """ 84 | 85 | @node(name="recall('{}')".format(key)) 86 | @bind_call_state 87 | def recaller(call_state, *_): 88 | if from_call_state: 89 | return from_call_state.data[key] 90 | return call_state.data[key] 91 | 92 | return recaller 93 | 94 | 95 | def concurrent(function: FunctionMerge) -> ConcurrentMerge: 96 | """ 97 | Upgrade the specified FunctionMerge object to a ConcurrentMerge, which runs each of its 98 | component functions in separate processes. See ConcurrentMerge documentation for more 99 | information. 100 | 101 | Usage: 102 | 103 | c = concurrent(long_running_function + other_long_running_function) 104 | c(input_data) # The two functions run in parallel 105 | """ 106 | return ConcurrentMerge(function) 107 | 108 | 109 | def mmap(function: tp.Callable, operator: tp.Callable = operators.concat) -> MergeMap: 110 | """ 111 | Upgrade the specified function to a MergeMap, which calls its single function once per input, 112 | as per the builtin `map` (https://docs.python.org/3.6/library/functions.html#map). 113 | 114 | Consider the name 'mmap' to be a placeholder for now. 115 | """ 116 | return MergeMap(MetaFunction.make_meta(function), operator) 117 | 118 | 119 | def locate_error( 120 | meta_function: MetaFunction, use_color=util.system_supports_color() 121 | ) -> SimpleFunction: 122 | """ 123 | Wrap the given MetaFunction with an error handler that adds location information to any 124 | exception raised therein. 125 | 126 | Usage: 127 | cmp = locate_error(a | b | c) 128 | cmp() 129 | """ 130 | 131 | def with_location(*args, call_state, **kwargs): 132 | new_e = None 133 | try: 134 | return meta_function(*args, call_state=call_state, **kwargs) 135 | except Exception as e: 136 | if hasattr(e, "location") and e.location: 137 | # If the exception has location info attached 138 | location = e.location 139 | else: 140 | location = call_state.highlight_active_function() 141 | 142 | if use_color: 143 | location = util.color_highlights(location) 144 | detailed_message = ( 145 | "{0!s} \n\nOccured in the following function: {1}".format(e, location) 146 | ) 147 | new_e = type(e)(detailed_message).with_traceback(e.__traceback__) 148 | new_e.__cause__ = e.__cause__ 149 | raise new_e 150 | 151 | with_location._receives_call_state = True 152 | return node(with_location, name=str(meta_function)) 153 | -------------------------------------------------------------------------------- /metafunctions/core/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import MetaFunction 2 | from .base import SimpleFunction 3 | from .base import DeferredValue 4 | from .base import FunctionChain 5 | from .base import FunctionMerge 6 | from .decorators import manage_call_state 7 | from .call_state import CallState 8 | from .concurrent import ConcurrentMerge 9 | -------------------------------------------------------------------------------- /metafunctions/core/base.py: -------------------------------------------------------------------------------- 1 | import typing as tp 2 | import abc 3 | import itertools 4 | import functools 5 | 6 | 7 | from metafunctions.core.decorators import binary_operation 8 | from metafunctions.core.decorators import manage_call_state 9 | from metafunctions.core.call_state import CallState 10 | from metafunctions import operators 11 | from metafunctions import exceptions 12 | 13 | 14 | class MetaFunction(metaclass=abc.ABCMeta): 15 | 16 | # Metafunctions will pass call state to any function with this attribute set to true 17 | _receives_call_state = True 18 | _function_join_str = "" 19 | 20 | @abc.abstractmethod 21 | def __init__(self, *args, **kwargs): 22 | """A MetaFunction is a function that contains other functions. When executed, it calls the 23 | functions it contains. 24 | """ 25 | self._functions = [] 26 | 27 | @abc.abstractmethod 28 | def __call__(self, *args, call_state=None, **kwargs): 29 | """Call the functions contained in this MetaFunction""" 30 | 31 | def __str__(self): 32 | return "({})".format( 33 | " {} ".format(self._function_join_str).join(str(f) for f in self.functions) 34 | ) 35 | 36 | @property 37 | def functions(self): 38 | return self._functions 39 | 40 | @staticmethod 41 | def make_meta(function): 42 | """Wrap the given function in a metafunction, unless it's already a metafunction""" 43 | if not isinstance(function, MetaFunction): 44 | return SimpleFunction(function) 45 | return function 46 | 47 | @staticmethod 48 | def defer_value(value): 49 | """Wrap the given value in a DeferredValue object, (which returns its value when called).""" 50 | return DeferredValue(value) 51 | 52 | @staticmethod 53 | def new_call_state(): 54 | return CallState() 55 | 56 | ### Operator overloads ### 57 | @binary_operation 58 | def __or__(self, other): 59 | return FunctionChain.combine(self, other) 60 | 61 | @binary_operation 62 | def __ror__(self, other): 63 | return FunctionChain.combine(other, self) 64 | 65 | @binary_operation 66 | def __and__(self, other): 67 | return FunctionMerge.combine(operators.concat, self, other) 68 | 69 | @binary_operation 70 | def __rand__(self, other): 71 | return FunctionMerge.combine(operators.concat, other, self) 72 | 73 | @binary_operation 74 | def __add__(self, other): 75 | return FunctionMerge(operators.add, (self, other)) 76 | 77 | @binary_operation 78 | def __radd__(self, other): 79 | return FunctionMerge(operators.add, (other, self)) 80 | 81 | @binary_operation 82 | def __sub__(self, other): 83 | return FunctionMerge(operators.sub, (self, other)) 84 | 85 | @binary_operation 86 | def __rsub__(self, other): 87 | return FunctionMerge(operators.sub, (other, self)) 88 | 89 | @binary_operation 90 | def __mul__(self, other): 91 | return FunctionMerge(operators.mul, (self, other)) 92 | 93 | @binary_operation 94 | def __rmul__(self, other): 95 | return FunctionMerge(operators.mul, (other, self)) 96 | 97 | @binary_operation 98 | def __truediv__(self, other): 99 | return FunctionMerge(operators.truediv, (self, other)) 100 | 101 | @binary_operation 102 | def __rtruediv__(self, other): 103 | return FunctionMerge(operators.truediv, (other, self)) 104 | 105 | @binary_operation 106 | def __matmul__(self, other): 107 | from metafunctions.api import star 108 | 109 | return FunctionChain.combine(self, star(other)) 110 | 111 | @binary_operation 112 | def __rmatmul__(self, other): 113 | from metafunctions.api import star 114 | 115 | return FunctionChain.combine(other, star(self)) 116 | 117 | 118 | class FunctionChain(MetaFunction): 119 | _function_join_str = "|" 120 | 121 | def __init__(self, *functions): 122 | """A FunctionChain is a metafunction that calls its functions in sequence, passing the 123 | results of the first function subsequent functions. 124 | """ 125 | super().__init__() 126 | self._functions = functions 127 | 128 | @manage_call_state 129 | def __call__(self, *args, **kwargs): 130 | f_iter = iter(self._functions) 131 | result = next(f_iter)(*args, **kwargs) 132 | for f in f_iter: 133 | result = f(result, **kwargs) 134 | return result 135 | 136 | def __repr__(self): 137 | return "{self.__class__.__name__}{self.functions}".format(self=self) 138 | 139 | @classmethod 140 | def combine(cls, *funcs): 141 | """Merge chains; i.e., combine all FunctionChains in `funcs` into a single FunctionChain.""" 142 | new_funcs = [] 143 | for f in funcs: 144 | if type(f) is cls: 145 | new_funcs.extend(f.functions) 146 | else: 147 | new_funcs.append(f) 148 | return cls(*new_funcs) 149 | 150 | 151 | class FunctionMerge(MetaFunction): 152 | _character_to_operator = { 153 | "+": operators.add, 154 | "-": operators.sub, 155 | "*": operators.mul, 156 | "/": operators.truediv, 157 | "&": operators.concat, 158 | } 159 | _operator_to_character = {v: k for k, v in _character_to_operator.items()} 160 | 161 | def __init__(self, merge_func: tp.Callable, functions: tuple, function_join_str=""): 162 | """ 163 | A FunctionMerge merges its functions by executing all of them and passing their results to 164 | `merge_func`. 165 | 166 | Behaviour of __call__: 167 | 168 | FunctionMerge does not pass all positional arguments to all of its functions. Rather, given 169 | `f=FunctionMerge()` when f is called with `f(*args)`, 170 | 171 | * if len(args) == 1, each component function is called with args[0] 172 | * if len(args) > 1 <= len(functions), function n is called with arg n. Any remaining 173 | functions after all args have been exhausted are called with no args. 174 | * if len(args) < len(functions), a MetaFunction CallError is raised. 175 | 176 | Args: 177 | function_join_str: If you're using a `merge_func` that is not one of the standard operator 178 | functions, use this argument to provide a custom character to use in string formatting. If 179 | not provided, we default to using str(merge_func). 180 | """ 181 | super().__init__() 182 | self._merge_func = merge_func 183 | self._functions = functions 184 | self._function_join_str = function_join_str or self._operator_to_character.get( 185 | merge_func, str(merge_func) 186 | ) 187 | 188 | @manage_call_state 189 | def __call__(self, *args, **kwargs): 190 | args_iter, func_iter = self._get_call_iterators(args) 191 | 192 | results = [] 193 | # Note that args_iter appears first in the zip. This is because I know its len is <= 194 | # len(func_iter) (I asserted so above). In zip, if the first iterator is longer than the 195 | # second, the first will be advanced one extra time, because zip has already called next() 196 | # on the first iterator before discovering that the second has been exhausted. 197 | for arg, f in zip(args_iter, func_iter): 198 | results.append(self._call_function(f, (arg,), kwargs)) 199 | 200 | # Any extra functions are called with no input 201 | results.extend([self._call_function(f, (), kwargs) for f in func_iter]) 202 | return self._merge_func(*results) 203 | 204 | def __repr__(self): 205 | return "{self.__class__.__name__}({self._merge_func}, {self.functions})".format( 206 | self=self 207 | ) 208 | 209 | @classmethod 210 | def combine(cls, merge_func: tp.Callable, *funcs, function_join_str=None): 211 | """Combine FunctionMerges. If consecutive FunctionMerges have the same merge_funcs, combine 212 | them into a single FunctionMerge. 213 | 214 | NOTE: combine does not check to make sure the merge_func can accept the new number of 215 | arguments, or that combining is appropriate for the operator. (e.g., it is inappropriate to 216 | combine FunctionMerges where order of operations matter. 5 / 2 / 3 != 5 / (2 / 3)) 217 | """ 218 | new_funcs = [] 219 | for f in funcs: 220 | if isinstance(f, cls) and f._merge_func is merge_func: 221 | new_funcs.extend(f.functions) 222 | else: 223 | new_funcs.append(f) 224 | return cls(merge_func, tuple(new_funcs), function_join_str=function_join_str) 225 | 226 | def _get_call_iterators(self, args): 227 | """Do length checking and return (`args_iter`, `call_iter`), iterables of arguments and 228 | self.functions. Call them using zip. Note that len(args) can be less than 229 | len(self.functions), and remaining functions should be called with no argument. 230 | """ 231 | args_iter = iter(args) 232 | func_iter = iter(self.functions) 233 | if len(args) > len(self.functions): 234 | raise exceptions.CallError( 235 | "{} takes 1 or <= {} arguments, but {} were given".format( 236 | self, len(self.functions), len(args) 237 | ) 238 | ) 239 | if len(args) == 1: 240 | args_iter = itertools.repeat(next(args_iter)) 241 | 242 | return args_iter, func_iter 243 | 244 | def _call_function(self, f, args: tuple, kwargs: dict): 245 | """This function receives one function, and the args and kwargs that should be used to call 246 | that function. It returns the result of the function call. This gets its own method so that 247 | subclasses can customize its behaviour. 248 | """ 249 | return f(*args, **kwargs) 250 | 251 | 252 | class SimpleFunction(MetaFunction): 253 | def __init__(self, function, name=None): 254 | """A MetaFunction-aware wrapper around a single function 255 | The `bind` parameter causes us to pass a meta object as the first argument to our inherited function, but it is only respected if the wrapped function is not another metafunction. 256 | """ 257 | # An interesting side effect of wraps: it causes simplefunctions to collapse into each 258 | # other. Because calling wraps on a function copies all that function's attributes to the 259 | # new function, we copy _function, etc from the wrapped function. Essentially 260 | # absorbing it. I'm not sure if that's good or bad. 261 | functools.wraps(function)(self) 262 | 263 | super().__init__() 264 | self._function = function 265 | self._name = name or getattr(function, "__name__", False) or str(function) 266 | 267 | @manage_call_state 268 | def __call__(self, *args, call_state, **kwargs): 269 | if getattr(self._function, "_receives_call_state", False): 270 | kwargs["call_state"] = call_state 271 | return self._function(*args, **kwargs) 272 | 273 | def __repr__(self): 274 | return "{self.__class__.__name__}({self.functions[0]!r})".format(self=self) 275 | 276 | def __str__(self): 277 | return self._name 278 | 279 | @property 280 | def functions(self): 281 | return (self._function,) 282 | 283 | 284 | class DeferredValue(SimpleFunction): 285 | def __init__(self, value): 286 | """A simple Deferred Value. Returns `value` when called. Equivalent to lambda x: x.""" 287 | self._value = value 288 | self._name = repr(value) 289 | 290 | def __call__(self, *args, **kwargs): 291 | return self._value 292 | 293 | def __repr__(self): 294 | return "{self.__class__.__name__}({self._value!r})".format(self=self) 295 | 296 | @property 297 | def functions(self): 298 | return (self,) 299 | -------------------------------------------------------------------------------- /metafunctions/core/call_state.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple, defaultdict, OrderedDict 2 | from metafunctions import util 3 | 4 | 5 | class CallState: 6 | Node = namedtuple("Node", "function insert_index") 7 | 8 | def __init__(self): 9 | """ 10 | A call tree keeps track of function execution order in metafunctions. This is used to 11 | accurately determine the location of the currently active function, and to identify 12 | exception locations. It can be thought of as a metafunction aware call stack. 13 | """ 14 | # Meta entry is conceptually the root node of the call tree 15 | self._meta_entry = None 16 | self.active_node = None 17 | self._nodes_visited = 0 18 | 19 | # A dictionary of {child: parent} functions 20 | self._parents = OrderedDict() 21 | 22 | # A dictionary of {parent: [children]}, in call order 23 | self._children = defaultdict(list) 24 | self.data = {} 25 | 26 | def push(self, f): 27 | """ 28 | Push a function onto the tree 29 | """ 30 | if self._meta_entry is None: 31 | node = self.Node(f, self._nodes_visited) 32 | self._meta_entry = node 33 | else: 34 | node = self.Node(f, self._nodes_visited) 35 | self._parents[node] = self.active_node 36 | self._children[self.active_node].append(node) 37 | self._nodes_visited += 1 38 | self.active_node = node 39 | 40 | def pop(self): 41 | """Remove last inserted f from the call tree.""" 42 | try: 43 | node, parent = self._parents.popitem() 44 | except KeyError: 45 | m = self._meta_entry 46 | self._meta_entry = None 47 | self.active_node = None 48 | return m 49 | self._children.pop(node, None) 50 | self.active_node = parent 51 | return node[0] 52 | 53 | def iter_parent_nodes(self, node): 54 | """ 55 | Return an iterator over all parents of this node in the tree, ending with the meta_entry 56 | point. 57 | """ 58 | # if node isn't in _parents, it must be the meta entry 59 | parent = self._parents.get(node, self._meta_entry) 60 | yield parent 61 | if parent is not self._meta_entry: 62 | yield from self.iter_parent_nodes(parent) 63 | 64 | def highlight_active_function(self): 65 | """ 66 | Return a formatted string showing the location of the most recently called function in 67 | call_state. 68 | 69 | Consider this a 'you are here' when called from within a function pipeline. 70 | """ 71 | current_function = self.active_node.function 72 | current_name = str(current_function) 73 | new_name = util.highlight(current_name) 74 | 75 | # rename active function in parent (if active function isn't in parent, active function 76 | # becomes parent) 77 | for parent in self.iter_parent_nodes(self.active_node): 78 | parent_name = str(parent.function) 79 | 80 | # Note we count occurences of current name in functions we've called so far. We do this 81 | # because it's possible previous functions contain the name of this function. 82 | name_count = sum( 83 | str(n.function).count(current_name) for n in self._children[parent] 84 | ) 85 | 86 | new_name = util.replace_nth(parent_name, current_name, name_count, new_name) 87 | current_function = parent.function 88 | current_name = parent_name 89 | 90 | # if new parent name hasn't changed (meaning it didn't contain the name we're 91 | # highlighting), highlight the parent name 92 | if new_name == parent_name: 93 | new_name = util.highlight(new_name) 94 | return new_name 95 | -------------------------------------------------------------------------------- /metafunctions/core/concurrent.py: -------------------------------------------------------------------------------- 1 | import os 2 | from operator import itemgetter 3 | from multiprocessing import Queue 4 | from collections import namedtuple 5 | import functools 6 | import pickle 7 | 8 | from metafunctions.core import FunctionMerge 9 | from metafunctions.core import manage_call_state 10 | from metafunctions import exceptions 11 | 12 | # Result tuple to be sent back from workers. Defined at module level for eas of pickling 13 | _ConcurrentResult = namedtuple( 14 | "_ConcurrentResult", "index result call_state_data exception location" 15 | ) 16 | 17 | 18 | class ConcurrentMerge(FunctionMerge): 19 | def __init__(self, function_merge: FunctionMerge): 20 | """A subclass of FunctionMerge that calls each of its component functions in parallel. 21 | 22 | ConcurrentMerge takes a FunctionMerge object and upgrades it. 23 | """ 24 | if not isinstance(function_merge, FunctionMerge): 25 | # This check is necessary because functools.wraps will copy FunctionMerge attributes to 26 | # objects that are not FunctionMerges, so this init will succeed, then result in errors 27 | # at call time. 28 | raise exceptions.CompositionError( 29 | "{} can only upgrade FunctionMerges".format(type(self)) 30 | ) 31 | if not hasattr(os, "fork"): 32 | raise exceptions.CompositionError( 33 | "{} requires os.fork, and thus is only available on unix".format( 34 | type(self).__name__ 35 | ) 36 | ) 37 | 38 | super().__init__( 39 | function_merge._merge_func, 40 | function_merge._functions, 41 | function_merge._function_join_str, 42 | ) 43 | self._function_merge = function_merge 44 | 45 | def __str__(self): 46 | merge_name = str(self._function_merge) 47 | return ( 48 | "concurrent{}".format(merge_name) 49 | if merge_name.startswith("(") 50 | else "concurrent({})".format(merge_name) 51 | ) 52 | 53 | @manage_call_state 54 | def __call__(self, *args, **kwargs): 55 | """We fork here, and execute each function in a child process before joining the results 56 | with _merge_func 57 | """ 58 | arg_iter, func_iter = self._get_call_iterators(args) 59 | enumerated_funcs = enumerate(func_iter) 60 | result_q = Queue() 61 | 62 | # spawn a child for each function 63 | children = [] 64 | for arg, (i, f) in zip(arg_iter, enumerated_funcs): 65 | child_pid = self._process_in_fork(i, f, result_q, (arg,), kwargs) 66 | children.append(child_pid) 67 | 68 | # iterate over any remaining functions for which we have no args 69 | for i, f in enumerated_funcs: 70 | child_pid = self._process_in_fork(i, f, result_q, (), kwargs) 71 | children.append(child_pid) 72 | 73 | # the parent waits for all children to complete 74 | for pid in children: 75 | os.waitpid(pid, 0) 76 | 77 | # then retrieves the results 78 | results = [] 79 | # iterate over the result q in sorted order 80 | result_q.put(None) 81 | for r in sorted(iter(result_q.get, None), key=itemgetter(0)): 82 | if r.exception: 83 | raise exceptions.ConcurrentException( 84 | "Caught exception in child process", location=r.location 85 | ) from pickle.loads(r.exception) 86 | kwargs["call_state"].data.update(pickle.loads(r.call_state_data)) 87 | results.append(pickle.loads(r.result)) 88 | return self._merge_func(*results) 89 | 90 | def _get_call_iterators(self, args): 91 | return self._function_merge._get_call_iterators(args) 92 | 93 | def _call_function(self, f, args: tuple, kwargs: dict): 94 | return self._function_merge._call_function(f, args, kwargs) 95 | 96 | def _process_in_fork(self, idx, func, result_q, args, kwargs): 97 | """Call self._call_function in a child process. This function returns the ID of the child 98 | in the parent process, while the child process calls _call_function, puts the results in 99 | the provided queues, then exits. 100 | """ 101 | pid = os.fork() 102 | if pid: 103 | return pid 104 | 105 | # here we are the child 106 | make_result = functools.partial( 107 | _ConcurrentResult, 108 | result=None, 109 | exception=None, 110 | index=idx, 111 | call_state_data=None, 112 | location="", 113 | ) 114 | 115 | result = None 116 | try: 117 | r = self._call_function(func, args, kwargs) 118 | 119 | # pickle here, so that we can't crash with pickle errors in the finally clause 120 | pickled_r = pickle.dumps(r) 121 | data = pickle.dumps(kwargs["call_state"].data) 122 | result = make_result(result=pickled_r, call_state_data=data) 123 | except Exception as e: 124 | try: 125 | # In case func does something stupid like raising an unpicklable exception 126 | pickled_exception = pickle.dumps(e) 127 | except AttributeError: 128 | pickled_exception = pickle.dumps( 129 | AttributeError("Unplicklable exception raised in {}".format(func)) 130 | ) 131 | result = make_result( 132 | exception=pickled_exception, 133 | location=kwargs["call_state"].highlight_active_function(), 134 | ) 135 | finally: 136 | result_q.put(result) 137 | # it's necessary to explicitly close the result_q and join its background thread here, 138 | # because the below os._exit won't allow time for any cleanup. 139 | result_q.close() 140 | result_q.join_thread() 141 | 142 | # This is the one place that the python docs say it's normal to use os._exit. Because 143 | # this is executed in a child process, calling sys.exit can have unintended 144 | # consequences. e.g., anything above this that catches the resulting SystemExit can 145 | # cause the child process to stay alive. the unittest framework does this. 146 | os._exit(0) 147 | -------------------------------------------------------------------------------- /metafunctions/core/decorators.py: -------------------------------------------------------------------------------- 1 | """ 2 | Internal decorators that are applied to MetaFunction methods, not functions. 3 | """ 4 | from functools import wraps 5 | from collections.abc import Callable 6 | 7 | 8 | def binary_operation(method): 9 | """Internal decorator to apply common type checking for binary operations""" 10 | 11 | @wraps(method) 12 | def new_operation(self, other): 13 | if isinstance(other, Callable): 14 | new_other = self.make_meta(other) 15 | else: 16 | new_other = self.defer_value(other) 17 | return method(self, new_other) 18 | 19 | return new_operation 20 | 21 | 22 | def manage_call_state(call_method): 23 | """Decorates the call method to insure call_state is present in kwargs, or create a new one. 24 | If the call state isn't active, we assume we are the meta entry point. 25 | """ 26 | 27 | @wraps(call_method) 28 | def with_call_state(self, *args, **kwargs): 29 | try: 30 | call_state = kwargs["call_state"] 31 | except KeyError: 32 | call_state = self.new_call_state() 33 | kwargs["call_state"] = call_state 34 | call_state.push(self) 35 | r = call_method(self, *args, **kwargs) 36 | call_state.pop() 37 | return r 38 | 39 | return with_call_state 40 | -------------------------------------------------------------------------------- /metafunctions/exceptions.py: -------------------------------------------------------------------------------- 1 | class MetaFunctionError(Exception): 2 | pass 3 | 4 | 5 | class CompositionError(MetaFunctionError, TypeError): 6 | "An exception that occureds when MetaFunctions are composed incorrectly" 7 | pass 8 | 9 | 10 | class CallError(MetaFunctionError, TypeError): 11 | "An exception that occures when a MetaFunction is called incorrectly" 12 | 13 | def __init__(self, *args, location: str = ""): 14 | self.location = location 15 | super().__init__(*args) 16 | 17 | def __repr__(self): 18 | return "{}({}, {})".format( 19 | self.__class__.__name__, 20 | self.args, 21 | "location='{}'".format(self.location) if self.location else "", 22 | ) 23 | 24 | 25 | class ConcurrentException(CallError): 26 | "Concurrent specific call errors (e.g., things that aren't picklable)" 27 | -------------------------------------------------------------------------------- /metafunctions/map.py: -------------------------------------------------------------------------------- 1 | import typing as tp 2 | import itertools 3 | 4 | from metafunctions.core.concurrent import FunctionMerge 5 | from metafunctions.operators import concat 6 | 7 | 8 | class MergeMap(FunctionMerge): 9 | def __init__(self, function: tp.Callable, merge_function: tp.Callable = concat): 10 | """ 11 | MergeMap is a FunctionMerge with only one function. When called, it behaves like the 12 | builtin `map` function and calls its function once per item in the iterable(s) it receives. 13 | """ 14 | super().__init__(merge_function, (function,)) 15 | 16 | def _get_call_iterators(self, args): 17 | """ 18 | Each element in args is an iterable. 19 | """ 20 | args_iter = zip(*args) 21 | 22 | # Note that EVERY element in the func iter will be called, so we need to make sure the 23 | # length of our iterator is the same as the shortest iterable we received. 24 | shortest_arg = min(args, key=len) 25 | func_iter = itertools.repeat(self.functions[0], len(shortest_arg)) 26 | return args_iter, func_iter 27 | 28 | def _call_function(self, f, args: tuple, kwargs: dict): 29 | """In MergeMap, args will be a single element tuple containing the args for this function.""" 30 | return f(*args[0], **kwargs) 31 | 32 | def __str__(self): 33 | return "mmap({self.functions[0]!s})".format(self=self) 34 | 35 | def __repr__(self): 36 | return "{self.__class__.__name__}({self.functions[0]}, merge_function={self._merge_func})".format( 37 | self=self 38 | ) 39 | -------------------------------------------------------------------------------- /metafunctions/operators.py: -------------------------------------------------------------------------------- 1 | """ 2 | Extra operators used by MetaFunctions 3 | """ 4 | from operator import add, sub, truediv, mul 5 | 6 | 7 | def concat(*args): 8 | "concat(1, 2, 3) -> (1, 2, 3)" 9 | return args 10 | -------------------------------------------------------------------------------- /metafunctions/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ForeverWintr/metafunctions/5e0df8c7bd1e70b50efd07ef01a1c06ba7946fa3/metafunctions/tests/__init__.py -------------------------------------------------------------------------------- /metafunctions/tests/simple_nodes.py: -------------------------------------------------------------------------------- 1 | """ 2 | Some simple nodes to use in tests. 3 | """ 4 | from metafunctions.api import node 5 | 6 | 7 | @node 8 | def a(x): 9 | return x + "a" 10 | 11 | 12 | @node() 13 | def b(x): 14 | return x + "b" 15 | 16 | 17 | @node() 18 | def c(x): 19 | return x + "c" 20 | 21 | 22 | @node 23 | def d(x): 24 | return x + "d" 25 | 26 | 27 | @node 28 | def e(x): 29 | return x + "e" 30 | -------------------------------------------------------------------------------- /metafunctions/tests/test_api.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | import functools 3 | 4 | from metafunctions.tests.util import BaseTestCase 5 | from metafunctions.tests.simple_nodes import * 6 | from metafunctions.api import store, recall, node, bind_call_state 7 | from metafunctions.core import SimpleFunction, CallState 8 | 9 | 10 | class TestUnit(BaseTestCase): 11 | def test_bind_call_state(self): 12 | """ 13 | If decorated with bind_call_state, the function receives the call state dictionary as its 14 | first argument. 15 | """ 16 | 17 | @node 18 | @bind_call_state 19 | def a_(call_state, x): 20 | self.assertIsInstance(call_state, CallState) 21 | call_state.data["a"] = "b" 22 | return x + "a" 23 | 24 | @node 25 | @bind_call_state 26 | def f(call_state, x): 27 | return x + call_state.data.get("a", "f") 28 | 29 | self.assertEqual(a("_"), "_a") 30 | self.assertEqual(f("_"), "_f") 31 | 32 | cmp = a_ | f 33 | self.assertEqual(cmp("_"), "_ab") 34 | cmp = f | a_ | a_ | f + f 35 | self.assertEqual(cmp("_"), "_faab_faab") 36 | 37 | def test_node_bracketless(self): 38 | """ 39 | I'm allowing the node decorator to be applied without calling because this is how both 40 | celery and function_pipes work. 41 | """ 42 | 43 | @node 44 | def a(x): 45 | return x + "a" 46 | 47 | @node() 48 | def b(x): 49 | return x + "b" 50 | 51 | self.assertIsInstance(a, SimpleFunction) 52 | self.assertIsInstance(b, SimpleFunction) 53 | self.assertEqual((b | a)("_"), "_ba") 54 | 55 | def test_store(self): 56 | state = CallState() 57 | abc = a | b | store("output") | c 58 | big = ( 59 | a 60 | | b 61 | | c + store("ab") + store("ab2") 62 | | store("abc") 63 | | recall("ab") + recall("ab2") 64 | | c + recall("abc") 65 | ) 66 | 67 | self.assertEqual(abc("_", call_state=state), "_abc") 68 | self.assertEqual(state.data["output"], "_ab") 69 | self.assertEqual(big("_"), "_ab_abc_abc_ab_ab") 70 | 71 | def test_recall(self): 72 | state = a.new_call_state() 73 | state.data["k"] = "secret" 74 | 75 | cmp = a + b | store("k") | c + recall("k") 76 | self.assertEqual(cmp("_"), "_a_bc_a_b") 77 | 78 | cmp = a + b | store("k") | c + recall("k") | recall("k", from_call_state=state) 79 | self.assertEqual(cmp("_"), "secret") 80 | 81 | def test_str_store(self): 82 | # this should be possible 83 | self.assertEqual(str(store("key")), "store('key')") 84 | 85 | def test_str_recall(self): 86 | self.assertEqual(str(recall("key")), "recall('key')") 87 | -------------------------------------------------------------------------------- /metafunctions/tests/test_at_operator.py: -------------------------------------------------------------------------------- 1 | from metafunctions.api import node 2 | from metafunctions.tests.util import BaseTestCase 3 | from metafunctions.tests.simple_nodes import * 4 | 5 | 6 | class TestUnit(BaseTestCase): 7 | def test_broadcast(self): 8 | @node 9 | def f(*args): 10 | return args 11 | 12 | cmp = (a | b) @ f 13 | self.assertEqual(cmp("_"), ("_", "a", "b")) 14 | 15 | rcmp = (1, 2, 3) @ f 16 | self.assertEqual(rcmp(), (1, 2, 3)) 17 | 18 | def test_str_repr(self): 19 | cmp = a @ b 20 | self.assertEqual(str(cmp), "(a | star(b))") 21 | self.assertEqual(repr(cmp), "FunctionChain{}".format((a, cmp.functions[1]))) 22 | 23 | cmp = a | a @ b * 5 / 7 | b & b @ a 24 | self.assertEqual( 25 | str(cmp), "(a | (((a | star(b)) * 5) / 7) | (b & (b | star(a))))" 26 | ) 27 | 28 | cmp = (1, 2, 3) @ a 29 | self.assertEqual(str(cmp), "((1, 2, 3) | star(a))") 30 | 31 | cmp = a @ (b & c) 32 | self.assertEqual(str(cmp), "(a | star(b & c))") 33 | self.assertEqual( 34 | repr(cmp), 35 | "FunctionChain({0!r}, SimpleFunction({1}))".format( 36 | a, cmp._functions[1]._function 37 | ), 38 | ) 39 | 40 | def test_upgrade_merge(self): 41 | aabbcc = (a & b & c) @ (a & b & c) 42 | self.assertEqual(aabbcc("_"), ("_aa", "_bb", "_cc")) 43 | -------------------------------------------------------------------------------- /metafunctions/tests/test_call_state.py: -------------------------------------------------------------------------------- 1 | from metafunctions.tests.util import BaseTestCase 2 | from metafunctions.tests.simple_nodes import * 3 | from metafunctions.api import bind_call_state 4 | from metafunctions.api import node 5 | from metafunctions.api import mmap 6 | from metafunctions.api import locate_error 7 | from metafunctions.core import CallState 8 | 9 | 10 | class TestUnit(BaseTestCase): 11 | def test_bind_call_state(self): 12 | # Bound functions receive call state 13 | @node 14 | @bind_call_state 15 | def f(call_state, x): 16 | self.assertIsInstance(call_state, CallState) 17 | call_state.data["h"] = "b" 18 | return x + "f" 19 | 20 | @node 21 | def g(x): 22 | return x + "g" 23 | 24 | @node 25 | @bind_call_state 26 | def h(call_state, x): 27 | self.assertIsInstance(call_state, CallState) 28 | return x + call_state.data["h"] 29 | 30 | fg = f | g | h 31 | self.assertEqual(fg("_"), "_fgb") 32 | 33 | def test_provide_call_state(self): 34 | # A call state you provide is passed to all functions 35 | @node 36 | @bind_call_state 37 | def f(call_state, x): 38 | self.assertIs(call_state, c) 39 | call_state.data["h"] = "d" 40 | return x + "f" 41 | 42 | @node 43 | def g(x): 44 | return x + "g" 45 | 46 | @node 47 | @bind_call_state 48 | def h(call_state, x): 49 | self.assertIs(call_state, c) 50 | return x + call_state.data["h"] 51 | 52 | fg = f | g | h 53 | 54 | c = CallState() 55 | self.assertEqual(fg("_", call_state=c), "_fgd") 56 | 57 | def test_add_remove(self): 58 | # imagine these are functions 59 | a, b, c, d = "a b c d".split() 60 | Node = CallState.Node 61 | 62 | tree = CallState() 63 | tree.push(a) 64 | tree.push(a) 65 | self.assertEqual(tree.pop(), a) 66 | self.assertDictEqual(tree._parents, {}) 67 | 68 | tree.push(b) 69 | tree.push(a) 70 | self.assertDictEqual( 71 | tree._parents, {Node(b, 2): Node(a, 0), Node(a, 3): Node(b, 2)} 72 | ) 73 | self.assertDictEqual( 74 | tree._children, 75 | { 76 | Node(a, 0): [Node(a, 1), Node(b, 2)], 77 | Node(b, 2): [Node(a, 3)], 78 | }, 79 | ) 80 | 81 | tree.push(b) 82 | self.assertDictEqual( 83 | tree._children, 84 | { 85 | Node(a, 0): [Node(a, 1), Node(b, 2)], 86 | Node(b, 2): [Node(a, 3)], 87 | Node(a, 3): [Node(b, 4)], 88 | }, 89 | ) 90 | self.assertDictEqual( 91 | tree._parents, 92 | { 93 | Node(b, 4): Node(a, 3), 94 | Node(a, 3): Node(b, 2), 95 | Node(b, 2): Node(a, 0), 96 | }, 97 | ) 98 | 99 | self.assertEqual(tree.pop(), b) 100 | self.assertEqual(tree.pop(), a) 101 | for f in a, b, c, d: 102 | tree.push(f) 103 | tree.pop() 104 | self.assertDictEqual( 105 | tree._children, 106 | { 107 | Node(a, 0): [Node(a, 1), Node(b, 2)], 108 | Node(b, 2): [ 109 | Node(a, 3), 110 | Node(a, 5), 111 | Node(b, 6), 112 | Node(c, 7), 113 | Node(d, 8), 114 | ], 115 | }, 116 | ) 117 | self.assertDictEqual( 118 | tree._parents, 119 | { 120 | Node(b, 2): Node(a, 0), 121 | }, 122 | ) 123 | 124 | def test_iter_parent_nodes(self): 125 | @node 126 | def l(*args): 127 | return [] 128 | 129 | @node 130 | @bind_call_state 131 | def return_parent_nodes(cs, *args): 132 | return list(cs.iter_parent_nodes(cs.active_node)) 133 | 134 | def nodes2str(nodes): 135 | return [(str(p.function), p.insert_index) for p in nodes] 136 | 137 | def get_parents(f): 138 | cs = CallState() 139 | return nodes2str(f("", call_state=cs)) 140 | 141 | self.assertEqual(return_parent_nodes(), [(return_parent_nodes, 0)]) 142 | 143 | # create some crazy compositions to get parents out of 144 | simple_chain = a | b | c | return_parent_nodes 145 | self.assertListEqual(get_parents(simple_chain), [(str(simple_chain), 0)]) 146 | 147 | merges = l + (l + (l + return_parent_nodes)) 148 | self.assertListEqual( 149 | get_parents(merges), 150 | [ 151 | ("(l + return_parent_nodes)", 4), 152 | ("(l + (l + return_parent_nodes))", 2), 153 | ("(l + (l + (l + return_parent_nodes)))", 0), 154 | ], 155 | ) 156 | 157 | def test_highlight_active_function(self): 158 | fmt_index = 7 159 | 160 | @node 161 | def ff(x): 162 | return x + "F" 163 | 164 | @node 165 | @bind_call_state 166 | def f(call_state, x): 167 | if call_state._nodes_visited == fmt_index: 168 | location_string = call_state.highlight_active_function() 169 | self.assertEqual( 170 | location_string, "(a | b | ff | f | f | ->f<- | f | f)" 171 | ) 172 | self.assertEqual(x, "_abFff") 173 | return x + "f" 174 | 175 | pipe = a | b | ff | f | f | f | f | f 176 | pipe("_") 177 | 178 | state = CallState() 179 | af = a + f 180 | af("_", call_state=state) 181 | with self.assertRaises(AttributeError): 182 | curr_f = state.highlight_active_function() 183 | 184 | def test_hightlight_active_function_no_parents(self): 185 | @node 186 | @bind_call_state 187 | def f(cs): 188 | return cs.highlight_active_function() 189 | 190 | self.assertEqual(f(), "->f<-") 191 | 192 | def test_highlight_active_function_multichar(self): 193 | # Don't fail on long named functions. This is a regression test 194 | @node 195 | def fail(x): 196 | if not x: 197 | 1 / 0 198 | return x - 1 199 | 200 | cmp = fail | fail + a 201 | color = locate_error(cmp, use_color=True) 202 | no_color = locate_error(cmp, use_color=False) 203 | with self.assertRaises(ZeroDivisionError) as e: 204 | color(1) 205 | self.assertTrue( 206 | e.exception.args[0].endswith("(fail | (\x1b[31m->fail<-\x1b[0m + a))") 207 | ) 208 | with self.assertRaises(ZeroDivisionError) as e: 209 | no_color(1) 210 | self.assertTrue(e.exception.args[0].endswith("(fail | (->fail<- + a))")) 211 | 212 | def test_highlight_with_map(self): 213 | @node 214 | def no_fail(x): 215 | return x 216 | 217 | @node 218 | def fail(*args): 219 | 1 / 0 220 | 221 | cmp = a + b | (c & no_fail & fail) 222 | mapper = locate_error(("aaaaa", "BBBBB") | mmap(cmp), use_color=False) 223 | with self.assertRaises(ZeroDivisionError) as e: 224 | mapper() 225 | self.assertEqual( 226 | str(e.exception), 227 | "division by zero \n\nOccured in the following function: " 228 | "(('aaaaa', 'BBBBB') | mmap(((a + b) | (c & no_fail & ->fail<-))))", 229 | ) 230 | 231 | def test_tricky_highlight(self): 232 | def wrapper(mf, new_name=None): 233 | @node(name=new_name or str(mf)) 234 | @bind_call_state 235 | def meta_wrapper(cs, *args, **kwargs): 236 | return mf(*args, call_state=cs, **kwargs) 237 | 238 | return meta_wrapper 239 | 240 | @node 241 | @bind_call_state 242 | def locator_string(cs, *args): 243 | return cs.highlight_active_function() 244 | 245 | cmp = a | b + c | locator_string 246 | 247 | tim = wrapper(cmp, new_name="tim") 248 | atim = a | tim 249 | self.assertEqual(str(tim), "tim") 250 | self.assertEqual(str(atim), "(a | tim)") 251 | self.assertEqual(atim("_"), "(a | ->tim<-)") 252 | -------------------------------------------------------------------------------- /metafunctions/tests/test_concurrent.py: -------------------------------------------------------------------------------- 1 | import platform 2 | import operator 3 | import unittest 4 | from unittest import mock 5 | import functools 6 | 7 | import colors 8 | 9 | from metafunctions.tests.util import BaseTestCase 10 | from metafunctions.tests.simple_nodes import * 11 | from metafunctions.api import node 12 | from metafunctions.api import bind_call_state 13 | from metafunctions.api import concurrent 14 | from metafunctions.api import mmap 15 | from metafunctions.api import store 16 | from metafunctions.api import star 17 | from metafunctions.api import locate_error 18 | from metafunctions.core.concurrent import ConcurrentMerge 19 | from metafunctions import operators 20 | from metafunctions.core import CallState 21 | from metafunctions.exceptions import ConcurrentException, CompositionError, CallError 22 | 23 | 24 | @unittest.skipIf( 25 | platform.system() == "Windows", "Concurrent isn't supported on windows" 26 | ) 27 | class TestIntegration(BaseTestCase): 28 | def test_basic(self): 29 | ab = a + b 30 | cab = ConcurrentMerge(ab) 31 | self.assertEqual(cab("_"), "_a_b") 32 | 33 | def test_exceptions(self): 34 | @node 35 | def fail(x): 36 | if not x: 37 | 1 / 0 38 | return x - 1 39 | 40 | cmp = locate_error(0 | ConcurrentMerge(fail - fail), use_color=True) 41 | 42 | with self.assertRaises(ConcurrentException) as e: 43 | cmp() 44 | self.assertIsInstance(e.exception.__cause__, ZeroDivisionError) 45 | self.assertEqual( 46 | str(e.exception), 47 | "Caught exception in child process \n\nOccured in the following function: " 48 | "(0 | concurrent({} - fail))".format(colors.red("->fail<-")), 49 | ) 50 | self.assertIsInstance(e.exception.__cause__, ZeroDivisionError) 51 | 52 | def test_consistent_meta(self): 53 | """ 54 | Every function in the pipeline recieves the same meta. 55 | """ 56 | 57 | @node 58 | @bind_call_state 59 | def f(call_state, x): 60 | self.assertIs(call_state._meta_entry.function, cmp) 61 | return 1 62 | 63 | @node() 64 | @bind_call_state 65 | def g(call_state, x): 66 | self.assertIs(call_state._meta_entry.function, cmp) 67 | return 1 68 | 69 | @node 70 | @bind_call_state 71 | def h(call_state, x): 72 | self.assertIs(call_state._meta_entry.function, cmp) 73 | return 1 74 | 75 | @node 76 | @bind_call_state 77 | def i(call_state, x): 78 | self.assertIs(call_state._meta_entry.function, cmp) 79 | return 1 80 | 81 | cmp = ConcurrentMerge(h + f + f / h + i - g) 82 | self.assertEqual(cmp(1), 3) 83 | 84 | self.assertEqual(cmp(1, call_state=cmp.new_call_state()), 3) 85 | 86 | # how do pretty tracebacks work in multiprocessing? 87 | 88 | def test_call(self): 89 | c = concurrent(a + b) 90 | self.assertEqual(c("_"), "_a_b") 91 | self.assertEqual(c("-", "_"), "-a_b") 92 | with self.assertRaises(CallError): 93 | c("_", "_", "_") 94 | 95 | @node 96 | def d(): 97 | return "d" 98 | 99 | abd = concurrent(a & b & d) 100 | self.assertEqual(abd("-", "_"), ("-a", "_b", "d")) 101 | 102 | def test_concurrent(self): 103 | c = concurrent(a + b) 104 | self.assertIsInstance(c, ConcurrentMerge) 105 | self.assertEqual(c("_"), "_a_b") 106 | 107 | def test_not_concurrent(self): 108 | # can only upgrade FunctionMerges 109 | 110 | with self.assertRaises(CompositionError): 111 | concurrent(a) 112 | with self.assertRaises(CompositionError): 113 | concurrent(a | b) 114 | 115 | def test_str_repr(self): 116 | cab = ConcurrentMerge(a + b) 117 | cmap = concurrent(mmap(a)) 118 | 119 | self.assertEqual( 120 | repr(cab), "ConcurrentMerge({0}, ({1!r}, {2!r}))".format(operator.add, a, b) 121 | ) 122 | self.assertEqual(str(cab), "concurrent(a + b)") 123 | self.assertEqual(str(cmap), "concurrent(mmap(a))") 124 | 125 | def test_basic_map(self): 126 | # We can upgrade maps to run in parallel 127 | banana = "bnn" | concurrent(mmap(a)) | "".join 128 | str_concat = operators.concat | node("".join) 129 | batman = concurrent(mmap(a, operator=str_concat)) 130 | self.assertEqual(banana(), "banana") 131 | self.assertEqual(batman("nnnn"), "nananana") 132 | 133 | def test_multi_arg_map(self): 134 | @node 135 | def f(*args): 136 | return args 137 | 138 | m = concurrent(mmap(f)) 139 | 140 | with self.assertRaises(CompositionError): 141 | # Because star returns a simple function, we can't upgrade it. 142 | starmap = concurrent(star(mmap(f))) 143 | # we have to wrap concurrent in star instead. 144 | starmap = star(concurrent(mmap(f))) 145 | 146 | mapstar = concurrent(mmap(star(f))) 147 | 148 | self.assertEqual(m([1, 2, 3], [4, 5, 6]), ((1, 4), (2, 5), (3, 6))) 149 | self.assertEqual(m([1, 2, 3]), ((1,), (2,), (3,))) 150 | 151 | with self.assertRaises(TypeError): 152 | starmap([1, 2, 3]) 153 | self.assertEqual(starmap([[1, 2, 3]]), m([1, 2, 3])) 154 | 155 | cmp = ([1, 2, 3], [4, 5, 6]) | starmap 156 | self.assertEqual(cmp(), ((1, 4), (2, 5), (3, 6))) 157 | 158 | cmp = ([1, 2, 3], [4, 5, 6]) | mapstar 159 | self.assertEqual(cmp(), ((1, 2, 3), (4, 5, 6))) 160 | 161 | def test_call_state(self): 162 | # Call state should be usable in concurrent chains 163 | chain_a = a | b | store("ab") 164 | chain_b = b | a | store("ba") 165 | cmp = concurrent(chain_a & chain_b) 166 | state = CallState() 167 | 168 | self.assertEqual(cmp("_", call_state=state), ("_ab", "_ba")) 169 | self.assertDictEqual(state.data, {"ab": "_ab", "ba": "_ba"}) 170 | 171 | # If call_state.data contains something that isn't pickleable, fail gracefully 172 | bad = [lambda: None] | store("o") 173 | cmp = concurrent(bad & bad) 174 | with self.assertRaises(ConcurrentException): 175 | cmp() 176 | 177 | def test_unpicklable_return(self): 178 | # Concurrent can't handle functions that return unpicklable objects. Raise a descriptive 179 | # exception 180 | @node 181 | def f(): 182 | return lambda: None 183 | 184 | cmp = concurrent(f & f) 185 | with self.assertRaises(ConcurrentException): 186 | cmp() 187 | 188 | def test_unpicklable_exception(self): 189 | # Don't let child processes crash, even if they do weird things like raise unpickleable 190 | # exceptions 191 | @node 192 | def f(): 193 | class BadException(Exception): 194 | pass 195 | 196 | raise BadException() 197 | 198 | cmp = concurrent(f + f) 199 | with self.assertRaises(ConcurrentException): 200 | cmp() 201 | 202 | @mock.patch("metafunctions.core.concurrent.os.fork", return_value=0) 203 | @mock.patch("metafunctions.core.concurrent.os._exit") 204 | @mock.patch("multiprocessing.queues.Queue.close") 205 | @mock.patch("multiprocessing.queues.Queue.join_thread") 206 | @mock.patch("metafunctions.core.concurrent.os.waitpid") 207 | def test_no_fork(self, mock_wait, mock_join, mock_close, mock_exit, mock_fork): 208 | # This test re-runs concurrent tests with forking disabled. Partially this is to 209 | # address my inability to get coverage.py to recognize the code covered by forked 210 | # processes, but it's also useful to have single process coverage of _process_in_fork to 211 | # detect errors that may be squelched by the interactions of multiple processes 212 | 213 | # Re-run all tests with fork patched 214 | this_test = self.id().split(".")[-1] 215 | for test_name in ( 216 | t for t in dir(self) if t.startswith("test_") and t != this_test 217 | ): 218 | method = getattr(self, test_name) 219 | print("calling, ", method) 220 | with self.subTest(name=test_name): 221 | method() 222 | 223 | @mock.patch("metafunctions.core.concurrent.hasattr", return_value=False) 224 | def test_windows(self, mock_hasattr): 225 | with self.assertRaises(CompositionError) as e: 226 | concurrent(a + a) 227 | self.assertEqual( 228 | str(e.exception), 229 | "ConcurrentMerge requires os.fork, and thus is only available on unix", 230 | ) 231 | -------------------------------------------------------------------------------- /metafunctions/tests/test_deferred_value.py: -------------------------------------------------------------------------------- 1 | from metafunctions.core import DeferredValue 2 | from metafunctions.tests.util import BaseTestCase 3 | 4 | 5 | class TestUnit(BaseTestCase): 6 | def test_call(self): 7 | self.assertEqual(DeferredValue(5)(), 5) 8 | a = object() 9 | b = DeferredValue(a) 10 | self.assertIs(b(), a) 11 | 12 | def test_str_repr(self): 13 | a = object() 14 | b = DeferredValue(a) 15 | self.assertEqual(repr(b), "DeferredValue({0!r})".format(a)) 16 | self.assertEqual(str(DeferredValue(5)), "5") 17 | self.assertEqual(repr(DeferredValue("a")), "DeferredValue('a')") 18 | 19 | def test_functions(self): 20 | # To fulfil the metafunction interface, DeferredValue.functions is a tuple containing self 21 | # This lets you access the deffered value with (f() for f in x.functions) 22 | a = object() 23 | b = DeferredValue(a) 24 | self.assertIs(b, b.functions[0]) 25 | -------------------------------------------------------------------------------- /metafunctions/tests/test_function_chain.py: -------------------------------------------------------------------------------- 1 | import operator 2 | 3 | from metafunctions.core import FunctionChain 4 | from metafunctions.core import FunctionMerge 5 | from metafunctions.core import SimpleFunction 6 | from metafunctions.tests.util import BaseTestCase 7 | 8 | 9 | class TestUnit(BaseTestCase): 10 | def test_str(self): 11 | c = FunctionChain(a, b, l) 12 | self.assertEqual(str(c), "(a | b | )") 13 | self.assertEqual(repr(c), "FunctionChain{}".format((a, b, l))) 14 | 15 | def test_call(self): 16 | c = FunctionChain(a, b, l) 17 | self.assertEqual(c("_"), "_abl") 18 | 19 | def test_combine(self): 20 | """Avoid nesting chains""" 21 | chain = FunctionChain(a, b, l) 22 | merge = FunctionMerge(operator.add, (a, b)) 23 | 24 | self.assertEqual( 25 | str(FunctionChain.combine(chain, chain)), 26 | "(a | b | | a | b | )", 27 | ) 28 | self.assertEqual(str(chain | chain), "(a | b | | a | b | )") 29 | self.assertEqual( 30 | str(chain | merge | chain), 31 | "(a | b | | (a + b) | a | b | )", 32 | ) 33 | 34 | 35 | @SimpleFunction 36 | def a(x): 37 | return x + "a" 38 | 39 | 40 | @SimpleFunction 41 | def b(x): 42 | return x + "b" 43 | 44 | 45 | l = SimpleFunction(lambda x: x + "l") 46 | -------------------------------------------------------------------------------- /metafunctions/tests/test_function_merge.py: -------------------------------------------------------------------------------- 1 | import operator 2 | import unittest 3 | 4 | from metafunctions.core import FunctionMerge 5 | from metafunctions.core import SimpleFunction 6 | from metafunctions.tests.util import BaseTestCase 7 | from metafunctions.operators import concat 8 | from metafunctions import exceptions 9 | from metafunctions.api import node, star 10 | 11 | 12 | class TestUnit(BaseTestCase): 13 | def test_str(self): 14 | cmp = FunctionMerge(operator.add, (a, b)) 15 | self.assertEqual(str(cmp), "(a + b)") 16 | self.assertEqual( 17 | repr(cmp), "FunctionMerge({}, {})".format(operator.add, (a, b)) 18 | ) 19 | 20 | def test_call(self): 21 | cmp = FunctionMerge(operator.add, (a, b)) 22 | self.assertEqual(cmp("_"), "_a_b") 23 | self.assertEqual(cmp("-", "_"), "-a_b") 24 | with self.assertRaises(exceptions.CallError): 25 | cmp("_", "_", "_") 26 | 27 | @SimpleFunction 28 | def d(): 29 | return "d" 30 | 31 | abd = a & b & d 32 | self.assertEqual(abd("-", "_"), ("-a", "_b", "d")) 33 | 34 | def test_format(self): 35 | cmp = FunctionMerge(operator.add, (a, b), function_join_str="tacos") 36 | self.assertEqual(str(cmp), "(a tacos b)") 37 | 38 | def test_non_binary(self): 39 | def my_concat(*args): 40 | return "".join(args) 41 | 42 | cmp = FunctionMerge(my_concat, (a, a, a, a)) 43 | self.assertEqual(cmp("_"), "_a_a_a_a") 44 | 45 | self.assertEqual(str(cmp), "(a {m} a {m} a {m} a)".format(m=my_concat)) 46 | 47 | d = FunctionMerge(my_concat, (b, b), function_join_str="q") 48 | self.assertEqual(str(d), "(b q b)") 49 | 50 | def test_join(self): 51 | # The first real non-binary function! Like the example above. 52 | 53 | cmp = a & a & "sweet as" 54 | 55 | self.assertTupleEqual(cmp("_"), ("_a", "_a", "sweet as")) 56 | self.assertEqual(str(cmp), "(a & a & 'sweet as')") 57 | self.assertEqual( 58 | repr(cmp), 59 | "FunctionMerge({}, {})".format(concat, (a, a, cmp._functions[-1])), 60 | ) 61 | 62 | # __rand__ works too 63 | a_ = "sweet as" & a 64 | self.assertEqual(a_("+"), ("sweet as", "+a")) 65 | 66 | abc = (a & (b & c)) | "".join 67 | self.assertEqual(abc("_"), "_a_b_c") 68 | 69 | def test_combine(self): 70 | # Only combine FunctionMerges that have the same MergeFunc 71 | add = a + b 72 | also_add = b + a 73 | div = a / b 74 | 75 | # This combined FunctionMerge will fail if called (because addition is binary, 76 | # operator.add only takes two args). I'm just using to combine for test purposes. 77 | abba = FunctionMerge.combine(operator.add, add, also_add) 78 | self.assertEqual(str(abba), "(a + b + b + a)") 79 | self.assertEqual( 80 | repr(abba), "FunctionMerge({}, {})".format(operator.add, (a, b, b, a)) 81 | ) 82 | 83 | ab_ba = FunctionMerge.combine(operator.sub, add, also_add) 84 | self.assertEqual(str(ab_ba), "((a + b) - (b + a))") 85 | self.assertEqual( 86 | repr(ab_ba), "FunctionMerge({}, {})".format(operator.sub, (add, also_add)) 87 | ) 88 | 89 | abab = FunctionMerge.combine(operator.add, add, div) 90 | self.assertEqual(str(abab), "(a + b + (a / b))") 91 | self.assertEqual( 92 | repr(abab), "FunctionMerge({}, {})".format(operator.add, (a, b, div)) 93 | ) 94 | 95 | def custom(): 96 | pass 97 | 98 | abba_ = FunctionMerge.combine(custom, add, also_add, function_join_str="<>") 99 | self.assertEqual(str(abba_), "((a + b) <> (b + a))") 100 | self.assertEqual( 101 | repr(abba_), "FunctionMerge({}, {})".format(custom, (add, also_add)) 102 | ) 103 | 104 | def my_concat(*args): 105 | return "".join(args) 106 | 107 | bb = FunctionMerge(my_concat, (b, b), function_join_str="q") 108 | aa = FunctionMerge(my_concat, (a, a), function_join_str="q") 109 | bbaa = FunctionMerge.combine(my_concat, bb, aa, function_join_str="q") 110 | self.assertEqual(str(bbaa), "(b q b q a q a)") 111 | self.assertEqual( 112 | repr(bbaa), "FunctionMerge({}, {})".format(my_concat, (b, b, a, a)) 113 | ) 114 | 115 | def test_len_mismatch(self): 116 | # If len(inputs) <= len(functions), call remaining functions with no args. 117 | @node 118 | def f(x=None): 119 | if x: 120 | return x + "f" 121 | return "F" 122 | 123 | cmp = (a & b) | star(f & f & f & f) 124 | self.assertEqual(cmp("_"), ("_af", "_bf", "F", "F")) 125 | 126 | # if len(inputs) > len(functions), fail. 127 | cmp = (a & b & c) | star(f + f) 128 | with self.assertRaises(exceptions.CallError): 129 | cmp("_") 130 | 131 | @unittest.skip("TODO") 132 | def test_binary_functions(self): 133 | # The issue here is that f + f + f + f is not converted to a single FunctionMerge. Rather 134 | # it becomes nested FunctionMerges: (((f + f) + f) + f). Ideally we would be able to 135 | # handle this. One potential solution is to 'flatten' the FunctionMerge, but this doesn't 136 | # work for functions that aren't commutative. E.g., (a / b / c) != (a / (b / c)). I'm 137 | # leaving this test for now as a todo. 138 | @node 139 | def f(x=None): 140 | if x: 141 | return x + "f" 142 | return "F" 143 | 144 | cmp = (a & b) | star(f + f + f + f) 145 | self.assertEqual(cmp("_"), "_af_bfFF") 146 | 147 | 148 | @SimpleFunction 149 | def a(x): 150 | return x + "a" 151 | 152 | 153 | @SimpleFunction 154 | def b(x): 155 | return x + "b" 156 | 157 | 158 | @node 159 | def c(x): 160 | return x + "c" 161 | 162 | 163 | l = SimpleFunction(lambda x: x + "l") 164 | -------------------------------------------------------------------------------- /metafunctions/tests/test_imports.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import random 3 | 4 | 5 | class TestUnit(unittest.TestCase): 6 | def test_api_imports(self): 7 | expected_names = [ 8 | "node", 9 | "bind_call_state", 10 | "star", 11 | "store", 12 | "recall", 13 | "concurrent", 14 | "mmap", 15 | "locate_error", 16 | ] 17 | random.shuffle(expected_names) 18 | for name in expected_names: 19 | exec("from metafunctions import {}".format(name)) 20 | -------------------------------------------------------------------------------- /metafunctions/tests/test_map.py: -------------------------------------------------------------------------------- 1 | from metafunctions.tests.util import BaseTestCase 2 | from metafunctions.tests.simple_nodes import * 3 | from metafunctions.api import node 4 | from metafunctions.api import star 5 | from metafunctions.api import mmap 6 | from metafunctions.map import MergeMap 7 | from metafunctions import operators 8 | 9 | 10 | class TestIntegration(BaseTestCase): 11 | def test_basic(self): 12 | banana = "bnn" | MergeMap(a) | "".join 13 | str_concat = operators.concat | node("".join) 14 | batman = MergeMap(a, merge_function=str_concat) 15 | self.assertEqual(banana(), "banana") 16 | self.assertEqual(batman("nnnn"), "nananana") 17 | 18 | def test_multi_arg(self): 19 | @node 20 | def f(*args): 21 | return args 22 | 23 | m = mmap(f) 24 | starmap = star(mmap(f)) 25 | mapstar = mmap(star(f)) 26 | 27 | self.assertEqual(m([1, 2, 3], [4, 5, 6]), ((1, 4), (2, 5), (3, 6))) 28 | self.assertEqual(m([1, 2, 3]), ((1,), (2,), (3,))) 29 | 30 | with self.assertRaises(TypeError): 31 | starmap([1, 2, 3]) 32 | self.assertEqual(starmap([[1, 2, 3]]), m([1, 2, 3])) 33 | 34 | cmp = ([1, 2, 3], [4, 5, 6]) | starmap 35 | self.assertEqual(cmp(), ((1, 4), (2, 5), (3, 6))) 36 | 37 | cmp = ([1, 2, 3], [4, 5, 6]) | mapstar 38 | self.assertEqual(cmp(), ((1, 2, 3), (4, 5, 6))) 39 | 40 | def test_auto_meta(self): 41 | mapsum = mmap(sum) 42 | self.assertEqual(mapsum([[1, 2], [3, 4]]), (3, 7)) 43 | self.assertEqual(str(mapsum), "mmap(sum)") 44 | 45 | def test_str_repr(self): 46 | m = MergeMap(a) 47 | self.assertEqual(str(m), "mmap(a)") 48 | self.assertEqual( 49 | repr(m), "MergeMap(a, merge_function={})".format(operators.concat) 50 | ) 51 | 52 | def test_loop(self): 53 | cmp = (b & c & "stoke") | mmap(a) 54 | self.assertEqual(cmp("_"), ("_ba", "_ca", "stokea")) 55 | 56 | def test_loop_with_non_meta(self): 57 | cmp = (b & c & "stoke") | mmap(len) 58 | self.assertEqual(cmp("_"), (2, 2, 5)) 59 | -------------------------------------------------------------------------------- /metafunctions/tests/test_pipe.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest import mock 3 | import functools 4 | import itertools 5 | 6 | from metafunctions.tests.util import BaseTestCase 7 | from metafunctions.tests.simple_nodes import * 8 | from metafunctions.api import node 9 | from metafunctions.api import locate_error 10 | from metafunctions.api import bind_call_state 11 | from metafunctions.core import CallState 12 | 13 | 14 | class TestIntegration(BaseTestCase): 15 | def test_basic_usage(self): 16 | self.assertEqual(a("_"), "_a") 17 | 18 | def test_wraps(self): 19 | @node 20 | def d(): 21 | "a docstring for d" 22 | 23 | self.assertEqual(d.__doc__, "a docstring for d") 24 | 25 | def test_auto_meta(self): 26 | """If possible, we upgrade functions to meta functions on the fly.""" 27 | 28 | def y(x): 29 | return x + "y" 30 | 31 | ay = a | y 32 | ya = y | a 33 | ayyyy = a | y | y | y | y 34 | 35 | # Can't do this 36 | # ayy = a | y + y 37 | 38 | # But this should work 39 | yayy = y | a + y 40 | yy_ya = y | y + a 41 | 42 | self.assertEqual(ay("_"), "_ay") 43 | self.assertEqual(ya("_"), "_ya") 44 | self.assertEqual(ayyyy("_"), "_ayyyy") 45 | self.assertEqual(yayy("_"), "_ya_yy") 46 | self.assertEqual(yy_ya("_"), "_yy_ya") 47 | 48 | def test_auto_meta_builtins(self): 49 | """We can upgrade builtin functions too""" 50 | 51 | mean = node(sum) / len 52 | self.assertEqual(mean((100, 200, 300)), 200) 53 | 54 | def test_basic_composition(self): 55 | composite = a | b | c | d 56 | self.assertEqual(composite("_"), "_abcd") 57 | 58 | def test_advanced_str(self): 59 | cmp = a | b + c + d | e 60 | self.assertEqual(str(cmp), "(a | ((b + c) + d) | e)") 61 | self.assertEqual(cmp("_"), "_ab_ac_ade") 62 | 63 | cmp = a + "f" 64 | self.assertEqual(str(cmp), "(a + 'f')") 65 | self.assertEqual( 66 | repr(cmp), 67 | "FunctionMerge(, " 68 | "(SimpleFunction({0!r}), DeferredValue('f')))".format(a._function), 69 | ) 70 | 71 | def test_non_callable_composition(self): 72 | """ 73 | Anything that is not callable in a composition is applied at call time (to the results 74 | of the composed functions). 75 | """ 76 | 77 | @node 78 | def g(x): 79 | return x 80 | 81 | cmps_to_expected = ( 82 | (g + 1, 11), 83 | (g - 1, 9), 84 | (g * 2, 20), 85 | (g / 2, 5), 86 | (1 + g, 11), 87 | (1 - g, -9), 88 | (2 * g, 20), 89 | (2 / g, 0.2), 90 | ) 91 | 92 | for cmp, expected in cmps_to_expected: 93 | with self.subTest(): 94 | self.assertEqual(cmp(10), expected) 95 | 96 | def test_single_calls(self): 97 | """every function is only called once""" 98 | call_count = 0 99 | 100 | @node 101 | def y(x): 102 | nonlocal call_count 103 | call_count += 1 104 | return x + "y" 105 | 106 | cmp = y | y * 2 | y + y | y 107 | self.assertEqual(cmp("_"), "_yy_yyy_yy_yyyy") 108 | self.assertEqual(call_count, 5) 109 | 110 | def test_repr(self): 111 | cmp = a | b | c | (lambda x: None) 112 | self.assertEqual(str(cmp), "(a | b | c | )") 113 | 114 | def test_called_functions(self): 115 | # This made more sense back before call_state was a tree. Consider removing. 116 | @node 117 | @bind_call_state 118 | def parent_test(call_state, x): 119 | return call_state 120 | 121 | ab = a | b 122 | abc = ab + c 123 | abc_ = abc | parent_test 124 | 125 | call_state = abc_("_") 126 | self.assertIsInstance(call_state, CallState) 127 | self.assertListEqual( 128 | [n.function for n in call_state._children[(abc_, 0)]], [abc, parent_test] 129 | ) 130 | 131 | def test_pretty_exceptions(self): 132 | @node 133 | def f(x): 134 | raise RuntimeError("Something bad happened!") 135 | 136 | @node 137 | def g(x): 138 | raise RuntimeError("Something bad happened in g!") 139 | 140 | abf = locate_error((a | b + f), use_color=False) 141 | abg = a | b + g 142 | 143 | with self.assertRaises(RuntimeError) as ctx: 144 | abf("_") 145 | 146 | self.assertEqual( 147 | str(ctx.exception), 148 | "Something bad happened! \n\nOccured in the following function: (a | (b + ->f<-))", 149 | ) 150 | 151 | # unprettified exceptions work 152 | with self.assertRaises(RuntimeError): 153 | abg("_") 154 | 155 | def test_consistent_meta(self): 156 | """ 157 | Every function in the pipeline recieves the same meta. 158 | """ 159 | 160 | @node 161 | @bind_call_state 162 | def f(call_state, x): 163 | self.assertIs(call_state._meta_entry.function, cmp) 164 | return 1 165 | 166 | @node() 167 | @bind_call_state 168 | def g(call_state, x): 169 | self.assertIs(call_state._meta_entry.function, cmp) 170 | return 1 171 | 172 | @node 173 | @bind_call_state 174 | def h(call_state, x): 175 | self.assertIs(call_state._meta_entry.function, cmp) 176 | return 1 177 | 178 | @node 179 | @bind_call_state 180 | def i(call_state, x): 181 | self.assertIs(call_state._meta_entry.function, cmp) 182 | return 1 183 | 184 | cmp = f | g | i | h + f + f / h + i - g 185 | self.assertEqual(cmp(1), 3) 186 | 187 | # this works if we provide our own call_state too. 188 | self.assertEqual(cmp(1, call_state=CallState()), 3) 189 | 190 | def test_consistent_meta_with_shared_state(self): 191 | @node 192 | @bind_call_state 193 | def f(call_state, expected): 194 | self.assertIs(call_state._meta_entry.function, expected) 195 | return expected 196 | 197 | cmpa = f & f 198 | cmpb = f | f | f 199 | 200 | cmpa(cmpa) 201 | cmpb(cmpb) 202 | 203 | state = CallState() 204 | cmpa(cmpa, call_state=state) 205 | self.assertDictEqual(state._parents, {}) 206 | cmpb(cmpb, call_state=state) 207 | self.assertEqual(state._nodes_visited, 7) 208 | 209 | def test_defaults(self): 210 | """ 211 | If you specify defaults in nodes, they are respected. 212 | """ 213 | 214 | @node 215 | def f(x="F"): 216 | return x + "f" 217 | 218 | @node 219 | @bind_call_state 220 | def g(call_state, x="G"): 221 | return x + "g" 222 | 223 | cmp = f | g | f + g 224 | self.assertEqual(cmp(), "FfgfFfgg") 225 | self.assertEqual(cmp("_"), "_fgf_fgg") 226 | 227 | cmp2 = g | f | f + g 228 | self.assertEqual(cmp2(), "GgffGgfg") 229 | self.assertEqual(cmp2("_"), "_gff_gfg") 230 | 231 | def test_complex_exceptions(self): 232 | @node 233 | def query_volume(x): 234 | return str(x**2) 235 | 236 | @node 237 | def query_price(x): 238 | return "$" + str(x**3) 239 | 240 | numeric = (query_volume | float) + (query_price | float) 241 | with self.assertRaises(ValueError) as e: 242 | numeric(2) 243 | 244 | def test_decoration(self): 245 | # It should be possible to decorate a metafunction with another metafunction and have 246 | # everything still work (as long as the decorated function gets upgraded to a metafunction). 247 | # I don't think this is something that will happen, but I want to enforce that it is 248 | # possible, for purposes of conceptual purity. 249 | @node() 250 | @bind_call_state 251 | def f(call_state, x): 252 | self.assertIs(call_state._meta_entry.function, abcf) 253 | return x + "f" 254 | 255 | fn = node(f + "sup") 256 | self.assertEqual(repr(fn), "SimpleFunction({0!r})".format((f + "sup"))) 257 | self.assertEqual(str(fn), "(f + 'sup')") 258 | abcf = a | b | c | fn 259 | 260 | self.assertEqual(abcf("_"), "_abcfsup") 261 | 262 | def test_kwargs(self): 263 | # Kwargs are passed to all functions 264 | 265 | @node 266 | def k(x, k="k"): 267 | return x + k 268 | 269 | self.assertEqual(k("_"), "_k") 270 | self.assertEqual(k("_", k="_"), "__") 271 | 272 | kk = k + k 273 | self.assertEqual(kk("_"), "_k_k") 274 | self.assertEqual(kk("_", k="_"), "____") 275 | 276 | klen = k | len 277 | self.assertEqual(klen("_"), 2) 278 | with self.assertRaises(TypeError): 279 | # passing a kwarg to len causes an error 280 | klen("_", k=5) 281 | 282 | def test_multi_call_immutability(self): 283 | # Unless you manually supply the same call_state, mutliple calls do not share state. 284 | @node 285 | @bind_call_state 286 | def f(call_state, x): 287 | if "f" not in call_state.data: 288 | call_state.data["f"] = x 289 | return x 290 | 291 | @node 292 | @bind_call_state 293 | def g(call_state, x): 294 | return x + call_state.data.get("f", "g") 295 | 296 | cmp = a | b | c | d | e | f | g 297 | self.assertEqual(cmp("_"), "_abcde_abcde") 298 | self.assertEqual(cmp("*"), "*abcde*abcde") 299 | 300 | state = CallState() 301 | self.assertEqual(cmp("_", call_state=state), "_abcde_abcde") 302 | self.assertEqual(cmp("*", call_state=state), "*abcde_abcde") 303 | 304 | def test_generators(self): 305 | # Can you use Metafunctions with generators? 306 | 307 | @node 308 | def gen_a(): 309 | yield from itertools.repeat("a") 310 | 311 | @node 312 | def gen_b(pred): 313 | yield from (x + "b" for x in pred) 314 | 315 | @node 316 | def gen_c(pred): 317 | return (x + "c" for x in pred) 318 | 319 | cmp = gen_a | gen_c | gen_b 320 | gen = cmp() 321 | for _ in range(5): 322 | self.assertEqual(next(gen), "acb") 323 | -------------------------------------------------------------------------------- /metafunctions/tests/test_simple_function.py: -------------------------------------------------------------------------------- 1 | from metafunctions.core import SimpleFunction 2 | from metafunctions.tests.util import BaseTestCase 3 | 4 | 5 | class TestUnit(BaseTestCase): 6 | def test_decorator(self): 7 | self.assertEqual(a.__name__, "a") 8 | self.assertEqual(a.__module__, __name__) 9 | 10 | def test_call(self): 11 | self.assertEqual(a("_"), "_a") 12 | self.assertEqual(l("_"), "_l") 13 | 14 | def test_str(self): 15 | self.assertEqual(repr(a), "SimpleFunction({0!r})".format(a._function)) 16 | self.assertEqual(str(a), "a") 17 | 18 | 19 | @SimpleFunction 20 | def a(x): 21 | return x + "a" 22 | 23 | 24 | l = SimpleFunction(lambda x: x + "l") 25 | -------------------------------------------------------------------------------- /metafunctions/tests/test_star.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | 4 | from metafunctions.api import node, star, concurrent 5 | from metafunctions.tests.simple_nodes import * 6 | from metafunctions import exceptions 7 | from metafunctions.tests.util import BaseTestCase 8 | 9 | 10 | class TestUnit(BaseTestCase): 11 | def test_simple_star(self): 12 | @node 13 | def f(*args): 14 | return args 15 | 16 | cmp = (a | b) | star(f) 17 | self.assertEqual(cmp("_"), ("_", "a", "b")) 18 | 19 | self.assertEqual(star(f)([1, 2, 3]), (1, 2, 3)) 20 | 21 | def test_str_repr(self): 22 | @node 23 | def f(*args): 24 | return args 25 | 26 | @star 27 | @node 28 | def g(*x): 29 | return x 30 | 31 | @star 32 | def h(*x): 33 | return x 34 | 35 | cmp = (a | b) | star(f) 36 | star_a = star(a) 37 | merge_star = star(a + b) 38 | chain_star = star(a | b) 39 | 40 | self.assertEqual(str(cmp), "(a | b | star(f))") 41 | self.assertEqual(str(star_a), "star(a)") 42 | self.assertEqual(str(g), "star(g)") 43 | self.assertEqual(str(merge_star), "star(a + b)") 44 | self.assertEqual(str(chain_star), "star(a | b)") 45 | 46 | # You can technically apply star to a regular function, and it'll become a SimpleFunction 47 | self.assertEqual( 48 | str(h), "star({})".format(h._function.__closure__[0].cell_contents) 49 | ) 50 | 51 | # reprs remain the same 52 | self.assertEqual(repr(star_a), "SimpleFunction({})".format(star_a._function)) 53 | 54 | @unittest.skipUnless(hasattr(os, "fork"), "Concurent isn't available on windows") 55 | def test_concurrent(self): 56 | # Concurrent and star can work together, although this organization no longer makes sense 57 | with self.assertRaises(exceptions.CompositionError): 58 | aabbcc = a & b & c | concurrent(star(a & b & c)) 59 | # self.assertEqual(aabbcc('_'), '_aa_bb_cc') 60 | 61 | aabbcc = (a & b & c) | star(concurrent(a & b & c)) 62 | self.assertEqual(aabbcc("_"), ("_aa", "_bb", "_cc")) 63 | 64 | @unittest.skip("TODO") 65 | def test_recursive_upgrade(self): 66 | aabbcc = (a & b & c) | star(a + b + c) 67 | self.assertEqual(aabbcc("_"), "_aa_bb_cc") 68 | 69 | def test_simple_args(self): 70 | @node 71 | def foo(a): 72 | return a 73 | 74 | @node 75 | def bar(b): 76 | return b 77 | 78 | @node 79 | def cool(a, b, c, d): 80 | return (a, b, c, d) 81 | 82 | f = (foo & bar & "arg_c" & "arg_d") | star(cool) 83 | assert f("a", "b") == ("a", "b", "arg_c", "arg_d") 84 | -------------------------------------------------------------------------------- /metafunctions/tests/test_util.py: -------------------------------------------------------------------------------- 1 | import colors 2 | 3 | from metafunctions.tests.util import BaseTestCase 4 | from metafunctions.tests.simple_nodes import * 5 | from metafunctions.api import node, locate_error 6 | from metafunctions import util 7 | 8 | 9 | class TestUnit(BaseTestCase): 10 | def test_locate_error(self): 11 | @node 12 | def fail(x): 13 | 1 / 0 14 | 15 | cmp = a + b | (c & fail & fail) 16 | with_tb = locate_error(cmp, use_color=False) 17 | 18 | with self.assertRaises(ZeroDivisionError) as e: 19 | cmp("x") 20 | self.assertEqual(str(e.exception), "division by zero") 21 | 22 | with self.assertRaises(ZeroDivisionError) as e: 23 | with_tb("x") 24 | self.assertEqual( 25 | str(e.exception), 26 | "division by zero \n\nOccured in the following function: ((a + b) | (c & ->fail<- & fail))", 27 | ) 28 | 29 | # regular calls still work 30 | with_tb2 = locate_error(a | b | c) 31 | self.assertEqual(with_tb2("_"), "_abc") 32 | 33 | self.assertEqual(str(with_tb), str(cmp)) 34 | self.assertEqual(str(with_tb), "((a + b) | (c & fail & fail))") 35 | self.assertEqual(str(with_tb2), "(a | b | c)") 36 | 37 | def test_replace_nth(self): 38 | s = "aaaaaaaaaa" 39 | self.assertEqual(util.replace_nth(s, "aa", 3, "BB"), "aaaaBBaaaa") 40 | 41 | new = util.replace_nth(s, "nothere", 1, "BB") 42 | self.assertEqual(new, s) 43 | 44 | def test_color_highlights(self): 45 | s = "a ->test<- string ->for<- highlight" 46 | self.assertEqual( 47 | util.color_highlights(s), 48 | "a {} string {} highlight".format( 49 | colors.red("->test<-"), colors.red("->for<-") 50 | ), 51 | ) 52 | -------------------------------------------------------------------------------- /metafunctions/tests/util.py: -------------------------------------------------------------------------------- 1 | """ 2 | Testing tools 3 | """ 4 | import unittest 5 | import tempfile 6 | import sys 7 | import random 8 | import os 9 | 10 | 11 | class BaseTestCase(unittest.TestCase): 12 | def __init__(self, methodName="runTest"): 13 | super().__init__(methodName) 14 | self._tempdir = None 15 | self.seed = random.randrange(sys.maxsize) 16 | 17 | @property 18 | def self_destructing_directory(self): 19 | """Return a temporary directory that exists for the duration of the test and is automatically removed after teardown.""" 20 | if not (self._tempdir and os.path.exists(self._tempdir.name)): 21 | self._tempdir = tempfile.TemporaryDirectory( 22 | prefix="{}_".format(self._testMethodName) 23 | ) 24 | self.addCleanup(self.cleanup_self_destructing_directory) 25 | self._tempdir.__enter__() 26 | return self._tempdir.name 27 | 28 | def cleanup_self_destructing_directory(self): 29 | # Only try to exit the temporaryDirectory context manager if the directory still exists 30 | if os.path.exists(self._tempdir.name): 31 | self._tempdir.__exit__(None, None, None) 32 | 33 | def _formatMessage(self, msg, standardMsg): 34 | msg = msg or "" 35 | return super()._formatMessage( 36 | msg="{} (seed: {})".format(msg, self.seed), standardMsg=standardMsg 37 | ) 38 | -------------------------------------------------------------------------------- /metafunctions/util.py: -------------------------------------------------------------------------------- 1 | """Utilities for working with MetaFunctions""" 2 | import os 3 | import sys 4 | import re 5 | 6 | DEFAULT_HIGHLIGHT_COLOR = None 7 | try: 8 | import colors 9 | except ImportError: 10 | _HAS_COLORS = False 11 | else: 12 | DEFAULT_HIGHLIGHT_COLOR = colors.red 13 | _HAS_COLORS = True 14 | 15 | HIGHLIGHT_TEMPLATE = "->{}<-" 16 | 17 | 18 | def highlight(string): 19 | return HIGHLIGHT_TEMPLATE.format(string) 20 | 21 | 22 | _color_regex = re.compile(HIGHLIGHT_TEMPLATE.format(".*?")) 23 | 24 | 25 | def color_highlights(string, color=DEFAULT_HIGHLIGHT_COLOR): 26 | """Color all highlights""" 27 | for substr in _color_regex.findall(string): 28 | string = string.replace(substr, color(substr)) 29 | return string 30 | 31 | 32 | def system_supports_color(): 33 | """ 34 | Returns True if the running system's terminal supports color, and False otherwise. Originally 35 | from Django, by way of StackOverflow: https://stackoverflow.com/a/22254892/1286571 36 | """ 37 | plat = sys.platform 38 | supported_platform = plat != "Pocket PC" and ( 39 | plat != "win32" or "ANSICON" in os.environ 40 | ) 41 | # isatty is not always implemented, #6223. 42 | is_a_tty = hasattr(sys.stdout, "isatty") and sys.stdout.isatty() 43 | if not _HAS_COLORS or not supported_platform or not is_a_tty: 44 | return False 45 | return True 46 | 47 | 48 | def replace_nth(string, substring, occurance_index: int, new_substring): 49 | """Return string, with the instance of substring at `occurance_index` replaced with new_substring""" 50 | escaped = re.escape(substring) 51 | # There's probably a better regex for this. 52 | regex = "((?:.*?{0}.*?){{{1}}}.*?){0}(.*$)".format(escaped, occurance_index - 1) 53 | return re.sub(regex, r"\1{}\2".format(new_substring), string) 54 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | tox 2 | black 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | MetaFunctions is a function composition and data pipelining library. 3 | 4 | For more information, please visit the `project on github `_. 5 | """ 6 | 7 | import os 8 | import sys 9 | import contextlib 10 | import pathlib 11 | import shutil 12 | from setuptools import setup, find_packages, Command 13 | 14 | import metafunctions 15 | 16 | here = os.path.abspath(os.path.dirname(__file__)) 17 | 18 | 19 | class UploadCommand(Command): 20 | """ 21 | Support setup.py upload. 22 | https://github.com/kennethreitz/setup.py/blob/master/setup.py 23 | """ 24 | 25 | description = "Build and publish the package." 26 | user_options = [] 27 | 28 | @staticmethod 29 | def status(s): 30 | """Prints things in bold.""" 31 | print("\033[1m{0}\033[0m".format(s)) 32 | 33 | def initialize_options(self): 34 | pass 35 | 36 | def finalize_options(self): 37 | pass 38 | 39 | def run(self): 40 | try: 41 | self.status("Removing previous builds…") 42 | shutil.rmtree(os.path.join(here, "dist")) 43 | except OSError: 44 | pass 45 | 46 | self.status("Building Source and Wheel (universal) distribution…") 47 | os.system("{0} setup.py sdist bdist_wheel --universal".format(sys.executable)) 48 | 49 | self.status("Uploading the package to PyPi via Twine…") 50 | os.system("twine upload dist/*") 51 | 52 | sys.exit() 53 | 54 | 55 | setup( 56 | name=metafunctions.__name__, 57 | version=metafunctions.__version__, 58 | description="Metafunctions is a function composition and data pipelining library", 59 | long_description=__doc__, 60 | url="https://github.com/ForeverWintr/metafunctions", 61 | author="Tom Rutherford", 62 | author_email="foreverwintr@gmail.com", 63 | license="MIT", 64 | # See https://pypi.python.org/pypi?%3Aaction=list_classifiers 65 | classifiers=[ 66 | "Development Status :: 5 - Production/Stable", 67 | "Intended Audience :: Developers", 68 | "Topic :: Software Development :: Libraries :: Python Modules", 69 | "License :: OSI Approved :: MIT License", 70 | "Programming Language :: Python :: 3.5", 71 | "Programming Language :: Python :: 3.6", 72 | "Programming Language :: Python :: 3.7", 73 | "Programming Language :: Python :: 3.8", 74 | "Programming Language :: Python :: 3.9", 75 | "Programming Language :: Python :: 3.10", 76 | "Programming Language :: Python :: 3.11", 77 | ], 78 | keywords="functional-programming function-composition", 79 | packages=find_packages(), 80 | test_suite="metafunctions.tests", 81 | install_requires="ansicolors>=1.1.8", 82 | # $ setup.py publish support. 83 | cmdclass={ 84 | "upload": UploadCommand, 85 | }, 86 | ) 87 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # tox (https://tox.readthedocs.io/) is a tool for running tests 2 | # in multiple virtualenvs. This configuration file will run the 3 | # test suite on all supported python versions. To use it, "pip install tox" 4 | # and then run "tox" from this directory. 5 | 6 | # Note: These versions are used when you run tox locally, but gh actions has its own env list. 7 | [tox] 8 | envlist = py35, py36, py37, py38, py39, py310 9 | 10 | [gh-actions] 11 | python = 12 | 3.6: py36 13 | 3.7: py37 14 | 3.8: py38 15 | 3.9: py39 16 | 3.10: py310 17 | 18 | 19 | [testenv] 20 | deps = 21 | coverage 22 | ansicolors 23 | commands = 24 | coverage run -pm unittest discover 25 | coverage combine 26 | coverage xml 27 | --------------------------------------------------------------------------------