├── test ├── __init__.py ├── test_infrastructure.py ├── test_lazy.py ├── test_util.py ├── test_react.py ├── test_functional.py ├── test_sources.py ├── test_sinks.py ├── test_reducers.py └── test_eager.py ├── requirements.txt ├── transducer ├── __init__.py ├── eager.py ├── coop.py ├── react.py ├── lazy.py ├── lazy_coop.py ├── _util.py ├── functional.py ├── sources.py ├── sinks.py ├── infrastructure.py ├── reducers.py └── transducers.py ├── docs ├── source │ ├── changes.rst │ ├── samples.rst │ ├── faq.rst │ ├── reference │ │ ├── reference.rst │ │ ├── eager.rst │ │ └── transducers.rst │ ├── differences.rst │ ├── front_matter │ │ └── front_matter.rst │ ├── narrative.rst │ ├── index.rst │ └── conf.py ├── Makefile └── make.bat ├── setup.cfg ├── .travis.yml ├── MANIFEST.in ├── .gitignore ├── LICENSE.txt ├── examples └── cooperative.py ├── CHANGES.txt ├── setup.py └── README.rst /test/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /test/test_infrastructure.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | cartouche>=1.1.2 2 | -------------------------------------------------------------------------------- /transducer/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.9' 2 | 3 | -------------------------------------------------------------------------------- /docs/source/changes.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../CHANGES.txt 2 | -------------------------------------------------------------------------------- /docs/source/samples.rst: -------------------------------------------------------------------------------- 1 | Samples 2 | ======= 3 | 4 | One day ... -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 0 3 | 4 | [metadata] 5 | license_file = LICENSE.txt 6 | -------------------------------------------------------------------------------- /docs/source/faq.rst: -------------------------------------------------------------------------------- 1 | Frequently Asked Questions 2 | ========================== 3 | 4 | So far, there are none. 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "3.3" 4 | - "3.4" 5 | - "3.5" 6 | - "3.6" 7 | # command to run tests 8 | script: nosetests 9 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE.txt 2 | include README.md 3 | recursive-include docs/build/html *.html *.css *.png *.js 4 | recursive-include test *.py 5 | recursive-include examples *.py 6 | -------------------------------------------------------------------------------- /docs/source/reference/reference.rst: -------------------------------------------------------------------------------- 1 | API Reference 2 | ============= 3 | 4 | ``transducer`` 5 | -------------- 6 | 7 | .. toctree:: 8 | :maxdepth: 3 9 | 10 | transducers 11 | eager 12 | -------------------------------------------------------------------------------- /docs/source/differences.rst: -------------------------------------------------------------------------------- 1 | Differences from Clojure's transducers 2 | ====================================== 3 | 4 | Although ``transducer`` is inspired by Clojure's transducers, there are 5 | inevitably some differences in order to accommodate the variance between Clojure 6 | and Python. 7 | 8 | -------------------------------------------------------------------------------- /docs/source/front_matter/front_matter.rst: -------------------------------------------------------------------------------- 1 | Copyright 2 | ========= 3 | 4 | ``transducer`` 5 | 6 | Copyright © 2014-2015 Sixty North 7 | 8 | Official Website 9 | ================ 10 | 11 | https://github.com/sixty-north/python-transducers 12 | 13 | License 14 | ======= 15 | 16 | .. include:: ../../../LICENCE.txt 17 | 18 | 19 | -------------------------------------------------------------------------------- /transducer/eager.py: -------------------------------------------------------------------------------- 1 | from transducer._util import UNSET 2 | from transducer.infrastructure import Reduced 3 | 4 | 5 | # Transducible processes 6 | 7 | def transduce(transducer, reducer, iterable, init=UNSET): 8 | r = transducer(reducer) 9 | accumulator = r.initial() if init is UNSET else init 10 | for item in iterable: 11 | accumulator = r.step(accumulator, item) 12 | if isinstance(accumulator, Reduced): 13 | accumulator = accumulator.value 14 | break 15 | return r.complete(accumulator) 16 | -------------------------------------------------------------------------------- /docs/source/narrative.rst: -------------------------------------------------------------------------------- 1 | ``transducer`` Introduction 2 | =========================== 3 | 4 | Current the best description of ``transducer`` can be found in our series of articles 5 | `Understanding Transducers Through Python `_. 6 | The code developed over the course of these articles is substantially 7 | the same as in this ``transducer`` package, although the package uses 8 | some further abstractions and tools which are largely irrelevant to 9 | understanding how transducers work. -------------------------------------------------------------------------------- /transducer/coop.py: -------------------------------------------------------------------------------- 1 | from transducer._util import UNSET 2 | from transducer.infrastructure import Reduced 3 | 4 | 5 | # Transducible processes 6 | 7 | async def transduce(transducer, reducer, aiterable, init=UNSET): 8 | r = transducer(reducer) 9 | accumulator = r.initial() if init is UNSET else init 10 | async for item in aiterable: 11 | accumulator = r.step(accumulator, item) 12 | if isinstance(accumulator, Reduced): 13 | accumulator = accumulator.value 14 | break 15 | return r.complete(accumulator) 16 | -------------------------------------------------------------------------------- /transducer/react.py: -------------------------------------------------------------------------------- 1 | from transducer._util import UNSET, coroutine 2 | from transducer.infrastructure import Reduced 3 | from transducer.reducers import sending 4 | 5 | @coroutine 6 | def transduce(transducer, target=UNSET): 7 | reducer = transducer(sending()) 8 | accumulator = target if target is not UNSET else reducer.initial() 9 | try: 10 | while True: 11 | item = (yield) 12 | accumulator = reducer.step(accumulator, item) 13 | if isinstance(accumulator, Reduced): 14 | accumulator = accumulator.value 15 | break 16 | assert accumulator is target 17 | except GeneratorExit: 18 | pass 19 | assert accumulator is target 20 | return reducer.complete(accumulator) 21 | -------------------------------------------------------------------------------- /docs/source/reference/eager.rst: -------------------------------------------------------------------------------- 1 | ``transducer.eager`` 2 | ==================== 3 | 4 | .. automodule:: transducer.eager 5 | 6 | Blah blah blah 7 | 8 | .. autosummary:: 9 | :nosignatures: 10 | 11 | .. currentmodule transducer.eager 12 | 13 | transduce 14 | 15 | .. autofunction:: transduce(transducer, reducer, iterating, init=UNSET) 16 | 17 | .. rubric:: Examples 18 | 19 | Eager transduction of a mapping over a list:: 20 | 21 | >>> from transducer.eager import transducer 22 | >>> from transducer.reducers import appending 23 | >>> from transducer.transducers import mapping 24 | >>> m = mapping(lambda x: x*x) 25 | >>> a = [1, 7, 9, 4, 3, 2] 26 | >>> transduce(mapping, appending(), a) 27 | [1, 49, 9, 4, 3, 2] 28 | 29 | 30 | -------------------------------------------------------------------------------- /transducer/lazy.py: -------------------------------------------------------------------------------- 1 | from collections import deque 2 | 3 | from transducer._util import pending_in 4 | from transducer.infrastructure import Reduced 5 | from transducer.reducers import appending 6 | 7 | 8 | 9 | # Transducible processes 10 | 11 | def transduce(transducer, iterable): 12 | r = transducer(appending()) 13 | accumulator = deque() 14 | reduced = False 15 | for item in iterable: 16 | accumulator = r.step(accumulator, item) 17 | if isinstance(accumulator, Reduced): 18 | accumulator = accumulator.value 19 | reduced = True 20 | 21 | yield from pending_in(accumulator) 22 | 23 | if reduced: 24 | break 25 | 26 | completed_result = r.complete(accumulator) 27 | assert completed_result is accumulator 28 | 29 | yield from pending_in(accumulator) 30 | -------------------------------------------------------------------------------- /transducer/lazy_coop.py: -------------------------------------------------------------------------------- 1 | from collections import deque 2 | 3 | from transducer.infrastructure import Reduced 4 | from transducer.reducers import appending 5 | 6 | 7 | # Transducible processes 8 | 9 | async def transduce(transducer, aiterable): 10 | r = transducer(appending()) 11 | accumulator = deque() 12 | reduced = False 13 | async for item in aiterable: 14 | accumulator = r.step(accumulator, item) 15 | if isinstance(accumulator, Reduced): 16 | accumulator = accumulator.value 17 | reduced = True 18 | 19 | while accumulator: 20 | yield accumulator.popleft() 21 | 22 | if reduced: 23 | break 24 | 25 | completed_result = r.complete(accumulator) 26 | assert completed_result is accumulator 27 | 28 | while accumulator: 29 | yield accumulator.popleft() 30 | -------------------------------------------------------------------------------- /test/test_lazy.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from transducer.functional import compose 3 | from transducer.lazy import transduce 4 | from transducer.transducers import (mapping, filtering, taking, dropping_while, distinct) 5 | 6 | 7 | class TestComposedTransducers(unittest.TestCase): 8 | 9 | def test_chained_transducers(self): 10 | result = transduce(transducer=compose( 11 | mapping(lambda x: x*x), 12 | filtering(lambda x: x % 5 != 0), 13 | taking(6), 14 | dropping_while(lambda x: x < 15), 15 | distinct()), 16 | iterable=range(20)) 17 | 18 | expected = [16, 36, 49] 19 | for r, e in zip(result, expected): 20 | self.assertEqual(r, e) 21 | 22 | if __name__ == '__main__': 23 | unittest.main() 24 | -------------------------------------------------------------------------------- /transducer/_util.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | 3 | 4 | # A sentinel for indicating unset function arguments in places 5 | # where None would be a legitimate value. 6 | UNSET = object() 7 | 8 | 9 | def pending_in(queue): 10 | """Yield items from the left of a queue. 11 | """ 12 | while queue: 13 | yield queue.popleft() 14 | 15 | 16 | def coroutine(func): 17 | """Decorator for priming generator-based coroutines. 18 | """ 19 | @wraps(func) 20 | def start(*args, **kwargs): 21 | g = func(*args, **kwargs) 22 | next(g) 23 | return g 24 | 25 | return start 26 | 27 | 28 | def prepend(item, iterable): 29 | yield item 30 | yield from iterable 31 | 32 | 33 | _EMPTY = tuple() 34 | 35 | 36 | def empty_iter(): 37 | return iter(_EMPTY) 38 | 39 | 40 | def iterator_or_none(iterator): 41 | try: 42 | first = next(iterator) 43 | except StopIteration: 44 | return None 45 | return prepend(first, iterator) 46 | -------------------------------------------------------------------------------- /test/test_util.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from transducer._util import prepend, empty_iter, iterator_or_none 3 | 4 | 5 | class TestPrepend(unittest.TestCase): 6 | 7 | def test_non_empty_iterable(self): 8 | result = list(prepend(42, [1, 2, 3])) 9 | self.assertListEqual(result, [42, 1, 2, 3]) 10 | 11 | def test_empty_iterable(self): 12 | result = list(prepend(42, [])) 13 | self.assertListEqual(result, [42]) 14 | 15 | 16 | class TestEmptyIter(unittest.TestCase): 17 | 18 | def test_is_empty(self): 19 | self.assertEqual(len(list(empty_iter())), 0) 20 | 21 | 22 | class TestIteratorOrNone(unittest.TestCase): 23 | 24 | def test_empty_iterator_returns_none(self): 25 | self.assertIsNone(iterator_or_none(empty_iter())) 26 | 27 | def test_non_empty_iterator_returns_iterator(self): 28 | items = [1, 4, 7, 2, 4] 29 | it = iter(items) 30 | remaining = iterator_or_none(it) 31 | result = list(remaining) 32 | self.assertListEqual(result, items) 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.gitignore.io 2 | 3 | ### Python ### 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | 8 | # PyCharm 9 | .idea/ 10 | 11 | # C extensions 12 | *.so 13 | 14 | # Distribution / packaging 15 | .Python 16 | env/ 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | lib/ 23 | lib64/ 24 | parts/ 25 | sdist/ 26 | var/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | MANIFEST 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .coverage 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | 57 | # Sphinx documentation 58 | docs/_build/ 59 | 60 | # PyBuilder 61 | target/ 62 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 Sixty North AS 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /transducer/functional.py: -------------------------------------------------------------------------------- 1 | from functools import reduce 2 | from itertools import chain 3 | 4 | 5 | def compose(f, *fs): 6 | """Compose functions right to left. 7 | 8 | compose(f, g, h)(x) -> f(g(h(x))) 9 | 10 | Args: 11 | f, *fs: The head and rest of a sequence of callables. The 12 | rightmost function passed can accept any arguments and 13 | the returned function will have the same signature as 14 | this last provided function. All preceding functions 15 | must be unary. 16 | 17 | Returns: 18 | The composition of the argument functions. The returned 19 | function will accept the same arguments as the rightmost 20 | passed in function. 21 | """ 22 | rfs = list(chain([f], fs)) 23 | rfs.reverse() 24 | 25 | def composed(*args, **kwargs): 26 | return reduce( 27 | lambda result, fn: fn(result), 28 | rfs[1:], 29 | rfs[0](*args, **kwargs)) 30 | 31 | return composed 32 | 33 | 34 | def identity(x): 35 | return x 36 | 37 | 38 | def true(*args, **kwargs): 39 | return True 40 | 41 | 42 | def false(*args, **kwargs): 43 | return False -------------------------------------------------------------------------------- /examples/cooperative.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from transducer.functional import compose 4 | from transducer.transducers import enumerating, windowing 5 | 6 | 7 | async def ticker(delay, to): 8 | """Yield numbers from 0 to `to` every `delay` seconds.""" 9 | for i in range(to): 10 | yield i 11 | await asyncio.sleep(delay) 12 | 13 | 14 | async def counter(delay, to): 15 | # from transducer.coop import transduce 16 | # from transducer.reducers import effecting 17 | # await transduce( 18 | # transducer=compose( 19 | # enumerating(), 20 | # windowing(3) 21 | # ), 22 | # reducer=effecting(print), 23 | # aiterable=ticker(delay, to) 24 | # ) 25 | 26 | from transducer.lazy_coop import transduce 27 | async for item in transduce( 28 | transducer=compose( 29 | enumerating(), 30 | windowing(3) 31 | ), 32 | aiterable=ticker(delay, to) 33 | ): 34 | print(item) 35 | 36 | 37 | def main(): 38 | loop = asyncio.get_event_loop() 39 | 40 | task = asyncio.ensure_future(counter(2.0, 20)) 41 | loop.run_until_complete(task) 42 | 43 | 44 | if __name__ == '__main__': 45 | main() 46 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. transducer documentation master file, created by 2 | sphinx-quickstart on Thu Feb 17 19:12:20 2011. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | ========== 7 | transducer 8 | ========== 9 | 10 | ``transducer`` is a Python package for ... 11 | 12 | It is licensed under the MIT License. 13 | 14 | Contents 15 | ======== 16 | 17 | Front Matter 18 | ------------ 19 | 20 | .. toctree:: 21 | :maxdepth: 2 22 | 23 | front_matter/front_matter 24 | 25 | Narrative Documentation 26 | ----------------------- 27 | 28 | Read this to learn how to use ``transducer``. 29 | 30 | .. toctree:: 31 | :maxdepth: 3 32 | 33 | narrative 34 | 35 | Reference Documentation 36 | ----------------------- 37 | 38 | Descriptions and examples for every public function, class and method in 39 | ``transducer``. 40 | 41 | .. toctree:: 42 | :maxdepth: 4 43 | 44 | reference/reference 45 | differences 46 | faq 47 | 48 | Detailed Change History 49 | ----------------------- 50 | 51 | .. toctree:: 52 | :maxdepth: 3 53 | 54 | changes 55 | 56 | Samples 57 | ------- 58 | 59 | More complex examples of non-trivial usage of ``transducer``: 60 | 61 | .. toctree:: 62 | :maxdepth: 3 63 | 64 | samples 65 | 66 | 67 | Indices and tables 68 | ================== 69 | 70 | * :ref:`genindex` 71 | * :ref:`modindex` 72 | * :ref:`search` 73 | 74 | -------------------------------------------------------------------------------- /transducer/sources.py: -------------------------------------------------------------------------------- 1 | import random 2 | from time import sleep 3 | from transducer._util import empty_iter, prepend 4 | 5 | 6 | def iterable_source(iterable, target): 7 | """Convert an iterable into a stream of events. 8 | 9 | Args: 10 | iterable: A series of items which will be sent to the target one by one. 11 | target: The target coroutine or sink. 12 | 13 | Returns: 14 | An iterator over any remaining items. 15 | """ 16 | it = iter(iterable) 17 | for item in it: 18 | try: 19 | target.send(item) 20 | except StopIteration: 21 | return prepend(item, it) 22 | return empty_iter() 23 | 24 | 25 | def poisson_source(rate, iterable, target): 26 | """Send events at random times with uniform probability. 27 | 28 | Args: 29 | rate: The average number of events to send per second. 30 | iterable: A series of items which will be sent to the target one by one. 31 | target: The target coroutine or sink. 32 | 33 | Returns: 34 | An iterator over any remaining items. 35 | """ 36 | if rate <= 0.0: 37 | raise ValueError("poisson_source rate {} is not positive".format(rate)) 38 | 39 | it = iter(iterable) 40 | for item in it: 41 | duration = random.expovariate(rate) 42 | sleep(duration) 43 | try: 44 | target.send(item) 45 | except StopIteration: 46 | return prepend(item, it) 47 | return empty_iter() 48 | -------------------------------------------------------------------------------- /test/test_react.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from transducer.functional import compose 3 | from transducer.react import transduce 4 | from transducer.sinks import CollectingSink, SingularSink 5 | from transducer.sources import iterable_source 6 | from transducer.transducers import (mapping, pairwise, filtering, first) 7 | 8 | 9 | class TestComposedTransducers(unittest.TestCase): 10 | 11 | def test_early_terminating_transducer(self): 12 | input = [0.0, 0.2, 0.8, 0.9, 1.1, 2.3, 2.6, 3.0, 4.1] 13 | output = SingularSink() 14 | 15 | iterable_source(iterable=input, 16 | target=transduce(first(lambda x: x > 1.0), 17 | target=output())) 18 | self.assertEqual(output.value, 1.1) 19 | 20 | def test_chained_transducers(self): 21 | input = [0.0, 0.2, 0.8, 0.9, 1.1, 2.3, 2.6, 3.0, 4.1] 22 | output = CollectingSink() 23 | 24 | iterable_source(iterable=input, 25 | target=transduce( 26 | compose(pairwise(), 27 | mapping(lambda p: p[1] - p[0]), 28 | filtering(lambda d: d < 0.5), 29 | mapping(lambda _: "double-click")), 30 | target=output())) 31 | 32 | result = list(output) 33 | self.assertListEqual(result, ['double-click', 'double-click', 'double-click', 'double-click', 'double-click']) 34 | 35 | 36 | if __name__ == '__main__': 37 | unittest.main() 38 | -------------------------------------------------------------------------------- /CHANGES.txt: -------------------------------------------------------------------------------- 1 | Changes 2 | ======= 3 | 4 | 5 | transducer 0.8 6 | -------------- 7 | 8 | Reorganised the code – particularly for the coroutine based event 9 | processing: sources, sinks and the react transduce now exist in their 10 | own modules. 11 | 12 | Clarified the semantics of the sources, which now no longer call 13 | close() on the targets. 14 | 15 | Single-sourced the version from the transducer/__init__.py file. 16 | 17 | Adds a new reducer for producing sets. 18 | 19 | A new reducer for completing regular Python reducing functions into 20 | objects supporting the full reducer protocol. 21 | 22 | Numerous fixes throughout the code as a result of vastly improved 23 | test coverage. 24 | 25 | 26 | transducer 0.7 27 | -------------- 28 | 29 | Rework to make the code closer to the code in the blog series. 30 | 31 | Much clearer reduction to single values for transducers such as first() 32 | and last() in conjunction with the expecting_single() reducer. 33 | 34 | Simplified and clarified much of the implementation. 35 | 36 | Dropped support for Python 3.2. 37 | 38 | 39 | transducer 0.6 40 | -------------- 41 | 42 | Correctness fixes by moving function locals in the transducer factories 43 | to be class attributes of the transducers. This allows transducers 44 | returned by the transducer factories to be safely used more than once. 45 | 46 | Renamed transducer.transducer to transducer.transducers. 47 | 48 | Relocate reducing functions to reducers.py 49 | 50 | Move transducer infrastructure to infrastructure.py 51 | 52 | 53 | transducer 0.5 54 | -------------- 55 | 56 | Initial release 57 | -------------------------------------------------------------------------------- /test/test_functional.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from transducer.functional import compose, true, identity, false 4 | 5 | 6 | class TestComposition(unittest.TestCase): 7 | 8 | def test_single(self): 9 | """ 10 | compose(f)(x) -> f(x) 11 | """ 12 | f = lambda x: x * 2 13 | c = compose(f) 14 | 15 | # We can't test the equivalence of functions completely, so... 16 | self.assertSequenceEqual([f(x) for x in range(1000)], 17 | [c(x) for x in range(1000)]) 18 | 19 | def test_double(self): 20 | """ 21 | compose(f, g)(x) -> f(g(x)) 22 | """ 23 | f = lambda x: x * 2 24 | g = lambda x: x + 1 25 | c = compose(f, g) 26 | 27 | self.assertSequenceEqual([f(g(x)) for x in range(100)], 28 | [c(x) for x in range(100)]) 29 | 30 | def test_triple(self): 31 | """ 32 | compose(f, g, h)(x) -> f(g(h(x))) 33 | """ 34 | f = lambda x: x * 2 35 | g = lambda x: x + 1 36 | h = lambda x: x - 7 37 | c = compose(f, g, h) 38 | 39 | self.assertSequenceEqual([f(g(h(x))) for x in range(100)], 40 | [c(x) for x in range(100)]) 41 | 42 | 43 | class TestFunctions(unittest.TestCase): 44 | 45 | def test_true(self): 46 | self.assertTrue(true()) 47 | 48 | def test_false(self): 49 | self.assertFalse(false()) 50 | 51 | def test_identity(self): 52 | self.assertEqual(identity(42), 42) 53 | 54 | 55 | if __name__ == '__main__': 56 | unittest.main() 57 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Transducer's setup.py 2 | 3 | import io 4 | import os 5 | import re 6 | 7 | from setuptools import setup 8 | 9 | 10 | def read(*names, **kwargs): 11 | with io.open( 12 | os.path.join(os.path.dirname(__file__), *names), 13 | encoding=kwargs.get("encoding", "utf8") 14 | ) as fp: 15 | return fp.read() 16 | 17 | 18 | def find_version(*file_paths): 19 | version_file = read(*file_paths) 20 | version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", 21 | version_file, re.M) 22 | if version_match: 23 | return version_match.group(1) 24 | raise RuntimeError("Unable to find version string.") 25 | 26 | 27 | with open('README.rst', 'r') as readme: 28 | long_description = readme.read() 29 | 30 | 31 | setup( 32 | name = "transducer", 33 | packages = ["transducer"], 34 | version = find_version("transducer/__init__.py"), 35 | description = "Transducers, similar to those in Clojure", 36 | author = "Sixty North AS", 37 | author_email = "rob@sixty-north.com", 38 | url = "https://github.com/sixty-north/python-transducers", 39 | keywords = ["Python", "functional"], 40 | license="MIT License", 41 | classifiers = [ 42 | "Development Status :: 3 - Alpha", 43 | "Programming Language :: Python", 44 | "Programming Language :: Python :: 3.5", 45 | "Programming Language :: Python :: 3.6", 46 | "Environment :: Other Environment", 47 | "Intended Audience :: Developers", 48 | "License :: OSI Approved :: MIT License", 49 | "Operating System :: OS Independent", 50 | "Topic :: Software Development :: Libraries :: Python Modules", 51 | "Topic :: Utilities", 52 | ], 53 | requires = [], 54 | long_description = long_description 55 | ) 56 | -------------------------------------------------------------------------------- /docs/source/reference/transducers.rst: -------------------------------------------------------------------------------- 1 | ``transducer.transducers`` 2 | ========================== 3 | 4 | .. automodule:: transducer.transducers 5 | 6 | Blah blah blah 7 | 8 | .. currentmodule transducer.transducers 9 | 10 | Transducer Factories 11 | -------------------- 12 | 13 | Call these functions to obtain transducer objects. 14 | 15 | 16 | .. autosummary:: 17 | :nosignatures: 18 | 19 | mapping 20 | 21 | 22 | .. autofunction:: mapping(transform) 23 | 24 | .. rubric:: Examples 25 | 26 | Mapping a squaring function over a list:: 27 | 28 | >>> from transducer.eager import transduce 29 | >>> from transducer.reducers import appending 30 | >>> from transducer.transducers import mapping 31 | >>> m = mapping(lambda x: x*x) 32 | >>> a = [1, 7, 9, 4, 3, 2] 33 | >>> transduce(m, appending(), a) 34 | [1, 49, 9, 4, 3, 2] 35 | 36 | 37 | .. autofunction:: filtering(predicate) 38 | 39 | .. rubric:: Examples 40 | 41 | Filtering even numbers from a list:: 42 | 43 | >>> from transducer.eager import transduce 44 | >>> from transducer.reducers import appending 45 | >>> from transducer.transducers import filtering 46 | >>> f = filtering(lambda x: x % 2 == 0) 47 | >>> a = [1, 7, 9, 4, 3, 2] 48 | >>> transduce(f, appending(), a) 49 | [4, 2] 50 | 51 | 52 | .. autofunction:: reducing(reducer, init=UNSET) 53 | 54 | .. rubric:: Examples 55 | 56 | Reducing a list of numbers to a single value by summing them:: 57 | 58 | >>> from transducer.eager import transduce 59 | >>> from transducer.reducers import expecting_single 60 | >>> from transducer.transducers import reducing 61 | >>> r = reducing(lambda x, y: x + y) 62 | >>> a = [1, 7, 9, 4, 3, 2] 63 | >>> transduce(r, expecting_single(), a) 64 | >>> 26 -------------------------------------------------------------------------------- /transducer/sinks.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Iterable, Sized 2 | from collections import deque 3 | import sys 4 | from transducer._util import coroutine, pending_in, UNSET 5 | 6 | 7 | @coroutine 8 | def null_sink(): 9 | while True: 10 | _ = (yield) 11 | 12 | 13 | @coroutine 14 | def rprint(sep='\n', end='\n', file=sys.stdout, flush=False): 15 | """A coroutine sink which prints received items stdout 16 | 17 | Args: 18 | sep: Optional separator to be printed between received items. 19 | end: Optional terminator to be printed after the last item. 20 | file: Optional stream to which to print. 21 | flush: Optional flag to force flushing after each item. 22 | """ 23 | try: 24 | first_item = (yield) 25 | file.write(str(first_item)) 26 | if flush: 27 | file.flush() 28 | while True: 29 | item = (yield) 30 | file.write(sep) 31 | file.write(str(item)) 32 | if flush: 33 | file.flush() 34 | except GeneratorExit: 35 | file.write(end) 36 | if flush: 37 | file.flush() 38 | 39 | 40 | class CollectingSink(Iterable, Sized): 41 | """Usage: 42 | 43 | sink = CollectingSink() 44 | 45 | # Do something to send items into the sink 46 | some_source(target=sink()) 47 | 48 | for item in sink: 49 | print(item) 50 | """ 51 | 52 | def __init__(self, maxlen=None): 53 | self._items = deque(maxlen=maxlen) 54 | 55 | @coroutine 56 | def __call__(self): 57 | while True: 58 | item = (yield) 59 | self._items.append(item) 60 | 61 | def __len__(self): 62 | return len(self._items) 63 | 64 | def __iter__(self): 65 | yield from pending_in(self._items) 66 | 67 | def clear(self): 68 | self._items.clear() 69 | 70 | 71 | class SingularSink: 72 | 73 | def __init__(self): 74 | self._item = UNSET 75 | 76 | @coroutine 77 | def __call__(self): 78 | while True: 79 | item = (yield) 80 | if self._item is not UNSET: 81 | break 82 | self._item = item 83 | 84 | @property 85 | def value(self): 86 | if self._item is UNSET: 87 | raise RuntimeError("Singular sink sent too few items.") 88 | return self._item 89 | 90 | @property 91 | def has_value(self): 92 | return self._item is not UNSET 93 | -------------------------------------------------------------------------------- /test/test_sources.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from transducer._util import iterator_or_none 3 | 4 | from transducer.sinks import CollectingSink, SingularSink 5 | from transducer.sources import iterable_source, poisson_source 6 | 7 | 8 | class TestIterableSource(unittest.TestCase): 9 | 10 | def test_empty_iterable_sends_no_items(self): 11 | collection = CollectingSink() 12 | sink = collection() 13 | iterable_source([], sink) 14 | self.assertEqual(len(collection), 0) 15 | 16 | def test_three_item_iterable_sends_three_items(self): 17 | items = [4, 7, 2, 1, 4] 18 | collection = CollectingSink() 19 | sink = collection() 20 | iterable_source(items, sink) 21 | result = list(collection) 22 | self.assertListEqual(result, items) 23 | 24 | def test_closed_target_exits_with_remaining_items(self): 25 | items = [4, 7, 2, 1, 4] 26 | collection = SingularSink() 27 | sink = collection() 28 | remaining = iterable_source(items, sink) 29 | self.assertListEqual(list(remaining), [7, 2, 1, 4]) 30 | 31 | def test_all_consumed_exits_with_empty_iterator(self): 32 | items = [4, 7, 2, 1, 4] 33 | collection = CollectingSink() 34 | sink = collection() 35 | remaining = iterable_source(items, sink) 36 | self.assertIsNone(iterator_or_none(remaining)) 37 | 38 | 39 | class TestPoissonSource(unittest.TestCase): 40 | 41 | def test_empty_iterable_sends_no_items(self): 42 | collection = CollectingSink() 43 | sink = collection() 44 | poisson_source(1e6, [], sink) 45 | self.assertEqual(len(collection), 0) 46 | 47 | def test_three_item_iterable_sends_three_items(self): 48 | items = [4, 7, 2, 1, 4] 49 | collection = CollectingSink() 50 | sink = collection() 51 | poisson_source(1e6, items, sink) 52 | result = list(collection) 53 | self.assertListEqual(result, items) 54 | 55 | def test_closed_target_exits_with_remaining_items(self): 56 | items = [4, 7, 2, 1, 4] 57 | collection = SingularSink() 58 | sink = collection() 59 | remaining = poisson_source(1e6, items, sink) 60 | self.assertListEqual(list(remaining), [7, 2, 1, 4]) 61 | 62 | def test_all_consumed_exits_with_empty_iterator(self): 63 | items = [4, 7, 2, 1, 4] 64 | collection = CollectingSink() 65 | sink = collection() 66 | remaining = poisson_source(1e6, items, sink) 67 | self.assertIsNone(iterator_or_none(remaining)) 68 | 69 | def test_non_positive_rate_raises_value_error(self): 70 | with self.assertRaises(ValueError): 71 | poisson_source(0.0, [], None) 72 | 73 | 74 | if __name__ == '__main__': 75 | unittest.main() 76 | -------------------------------------------------------------------------------- /transducer/infrastructure.py: -------------------------------------------------------------------------------- 1 | """Infrastructure for implementing transducers.""" 2 | 3 | from abc import ABCMeta, abstractmethod 4 | 5 | 6 | class Reduced: 7 | """A sentinel 'box' used to return the final value of a reduction. 8 | 9 | Wrapping a value in a Reduced instance is idempotent, so 10 | Reduced(Reduced(item)) is equivalent to Reduced(item) 11 | """ 12 | 13 | def __new__(cls, value): 14 | if isinstance(value, cls): 15 | return value 16 | obj = super().__new__(cls) 17 | obj._value = value 18 | return obj 19 | 20 | @property 21 | def value(self): 22 | return self._value 23 | 24 | def __repr__(self): 25 | return '{}({!r})'.format(self.__class__.__name__, self._value) 26 | 27 | 28 | class Reducer(object, metaclass=ABCMeta): 29 | 30 | @abstractmethod 31 | def initial(self): 32 | raise NotImplementedError 33 | 34 | @abstractmethod 35 | def step(self, result, item): 36 | raise NotImplementedError 37 | 38 | def complete(self, result): 39 | return result 40 | 41 | def __call__(self, result, item): 42 | """Reducing objects are callable so they can be used like functions.""" 43 | return self.step(result, item) 44 | 45 | 46 | class Transducer(Reducer): 47 | """An Base Class for Transducers which also serves as the identity transducer. 48 | """ 49 | 50 | def __init__(self, reducer): 51 | self._reducer = reducer 52 | 53 | def initial(self): 54 | return self._reducer.initial() 55 | 56 | def step(self, result, item): 57 | """Reduce one item. 58 | 59 | Called once for each item. Overrides should invoke the callable self._reducer 60 | directly as self._reducer(...) rather than as self._reducer.step(...) so that 61 | any 2-arity reduction callable can be used. 62 | 63 | Args: 64 | result: The reduced result thus far. 65 | item: The new item to be combined with result to give the new result. 66 | 67 | Returns: 68 | The newly reduced result; that is, result combined in some way with 69 | item to produce a new result. If reduction needs to be terminated, 70 | this method should return the sentinel Reduced(result). 71 | """ 72 | return self._reducer(result, item) 73 | 74 | def complete(self, result): 75 | """Called at exactly once when reduction is complete. 76 | 77 | Called on completion of a transducible process. The default implementation calls complete() 78 | on the underlying reducer, which should be done to meet the requirements of the transducer 79 | contract. Overrides of this method are the right place to deal with any pending state or 80 | perform with other clean-up actions. 81 | 82 | Args: 83 | result: The result prior to completion. 84 | 85 | Returns: 86 | The completed result. 87 | """ 88 | return self._reducer.complete(result) 89 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ===================== 2 | Transducers in Python 3 | ===================== 4 | 5 | This is a port of the *transducer* concept from Clojure to Python, 6 | with an emphasis on providing as Pythonic as interpretation of 7 | transducers as possible, rather than reproducing Clojurisms more 8 | literally. 9 | 10 | Installable Python package 11 | ========================== 12 | 13 | This package is available on the Python Package Index (PyPI) as 14 | `transducer `_. 15 | 16 | Status 17 | ====== 18 | 19 | Build status: 20 | 21 | .. image:: https://travis-ci.org/sixty-north/python-transducers.svg?branch=master 22 | :target: https://travis-ci.org/sixty-north/python-transducers 23 | :alt: Build Status 24 | 25 | .. image:: https://readthedocs.org/projects/python-transducers/badge/?version=latest 26 | :target: https://readthedocs.org/projects/python-transducers/?badge=latest 27 | :alt: Documentation Status 28 | 29 | Note: Documentation is very much a work in progress. 30 | 31 | What are transducers? 32 | ===================== 33 | 34 | Transducers are functions which transform reducers - hence the name. 35 | A reducer, in this case, is any function which you could pass to the 36 | ``reduce()`` function in the Python Standard Library ``functools`` 37 | module. Such reducers accept an initial or intermediate result and 38 | combine a new value with that result to produce a new (or updated) 39 | result. Transducers provide us with a convenient means to compose 40 | simple reducers into more complex and capable reducers. 41 | 42 | Furthermore, transducers facilitate the clean separation of 43 | concerns of how source values are input, how they are 44 | processed by reducers, and the how results output. This allows the 45 | same transducers to be (re)used with many sources and destinations 46 | of data, not just with iterable series. 47 | 48 | Transducers were developed by Rich Hickey, the driving force behind 49 | the Clojure programming language, and this package aims to bring 50 | the benefits of transducers to Python 3, whilst transforming some of 51 | the Clojurisms into more Pythonic solutions. 52 | 53 | An extended write-up of the development of Python transducers from 54 | scratch can be found in our series of articles 55 | `Understanding Transducers Through Python `_. 56 | The code developed over the course of these articles is substantially 57 | the same as in this ``transducer`` package, although the package uses 58 | some further abstractions and tools which are largely irrelevant to 59 | understanding how transducers work. 60 | 61 | This package, implements simple infrastructure for implementing 62 | transducers in Python, a selection of transducer implementations of 63 | common operations, and some 'transducible processes' which allow us 64 | to apply transducers to iterable series (both eagerly and lazily) and 65 | to use transducers to process 'push' events implemented as Python 66 | coroutines. 67 | -------------------------------------------------------------------------------- /transducer/reducers.py: -------------------------------------------------------------------------------- 1 | from transducer.infrastructure import Reducer, Reduced 2 | from transducer.sinks import null_sink 3 | 4 | 5 | class Appending(Reducer): 6 | 7 | def initial(self): 8 | return [] 9 | 10 | def step(self, result, item): 11 | result.append(item) 12 | return result 13 | 14 | _appending = Appending() 15 | 16 | 17 | def appending(): 18 | return _appending 19 | 20 | 21 | class Conjoining(Reducer): 22 | 23 | def initial(self): 24 | return tuple() 25 | 26 | def step(self, result, item): 27 | return result + type(result)((item,)) 28 | 29 | _conjoining = Conjoining() 30 | 31 | 32 | def conjoining(): 33 | return _conjoining 34 | 35 | 36 | class Adding(Reducer): 37 | 38 | def initial(self): 39 | return set() 40 | 41 | def step(self, result, item): 42 | result.add(item) 43 | return result 44 | 45 | _adding = Adding() 46 | 47 | 48 | def adding(): 49 | return _adding 50 | 51 | 52 | class Joining(Reducer): 53 | 54 | def __init__(self, separator): 55 | self._separator = separator 56 | 57 | def initial(self): 58 | return [] 59 | 60 | def step(self, result, item): 61 | result.append(item) 62 | return result 63 | 64 | def complete(self, result): 65 | return self._separator.join(result) 66 | 67 | 68 | class ExpectingSingle(Reducer): 69 | 70 | def __init__(self): 71 | self._num_steps = 0 72 | 73 | def initial(self): 74 | return None 75 | 76 | def step(self, result, item): 77 | self._num_steps += 1 78 | if self._num_steps > 1: 79 | raise RuntimeError("Too many steps!") 80 | assert result is None 81 | return item 82 | 83 | def complete(self, result): 84 | if self._num_steps < 1: 85 | raise RuntimeError("Too few steps!") 86 | return result 87 | 88 | 89 | def expecting_single(): 90 | return ExpectingSingle() 91 | 92 | 93 | class Sending(Reducer): 94 | 95 | def initial(self): 96 | return null_sink() 97 | 98 | def step(self, result, item): 99 | try: 100 | result.send(item) 101 | except StopIteration: 102 | return Reduced(result) 103 | else: 104 | return result 105 | 106 | def complete(self, result): 107 | result.close() 108 | return result 109 | 110 | _sending = Sending() 111 | 112 | 113 | def sending(): 114 | return _sending 115 | 116 | 117 | class Completing(Reducer): 118 | 119 | def __init__(self, reducer, identity): 120 | self._reducer = reducer 121 | self._identity = identity 122 | 123 | def initial(self): 124 | return self._identity 125 | 126 | def step(self, result, item): 127 | return self._reducer(result, item) 128 | 129 | 130 | def completing(reducer, identity=None): 131 | """Complete a regular reducing function to support the Reducer protocol. 132 | 133 | Args: 134 | reducer: A reducing function. e.g. lambda x, y: x+y 135 | identity: The identity (i.e. seed) value for reducer. e.g. zero 136 | 137 | Returns: 138 | An instance of the Completing reducer. 139 | """ 140 | 141 | return Completing(reducer, identity) 142 | 143 | 144 | class Effecting(Reducer): 145 | 146 | def __init__(self, f): 147 | if not callable(f): 148 | raise TypeError("{f} is not callable".format(f=f)) 149 | self._f = f 150 | 151 | def initial(self): 152 | return None 153 | 154 | def step(self, result, item): 155 | return self._f(item) 156 | 157 | 158 | def effecting(f): 159 | """Perform a non-pure side-effect by invoking a callable. 160 | 161 | Args: 162 | f: A unary function to which the current item will be passed. 163 | Any return value from this function will be used as the 164 | next intermediate result. Note that this function does 165 | not accept the intermediate result, so it is not a 166 | reducing function. As such, this transducer is mostly 167 | useful for invoking effectful functions such as print. 168 | 169 | Returns: 170 | An instance of the Effecting reducer. 171 | """ 172 | return Effecting(f) -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 14 | 15 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest 16 | 17 | help: 18 | @echo "Please use \`make ' where is one of" 19 | @echo " html to make standalone HTML files" 20 | @echo " dirhtml to make HTML files named index.html in directories" 21 | @echo " singlehtml to make a single large HTML file" 22 | @echo " pickle to make pickle files" 23 | @echo " json to make JSON files" 24 | @echo " htmlhelp to make HTML files and a HTML help project" 25 | @echo " qthelp to make HTML files and a qthelp project" 26 | @echo " devhelp to make HTML files and a Devhelp project" 27 | @echo " epub to make an epub" 28 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 29 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 30 | @echo " text to make text files" 31 | @echo " man to make manual pages" 32 | @echo " changes to make an overview of all changed/added/deprecated items" 33 | @echo " linkcheck to check all external links for integrity" 34 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 35 | 36 | clean: 37 | -rm -rf $(BUILDDIR)/* 38 | 39 | html: 40 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 41 | @echo 42 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 43 | 44 | dirhtml: 45 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 48 | 49 | singlehtml: 50 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 51 | @echo 52 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 53 | 54 | pickle: 55 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 56 | @echo 57 | @echo "Build finished; now you can process the pickle files." 58 | 59 | json: 60 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 61 | @echo 62 | @echo "Build finished; now you can process the JSON files." 63 | 64 | htmlhelp: 65 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 66 | @echo 67 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 68 | ".hhp project file in $(BUILDDIR)/htmlhelp." 69 | 70 | qthelp: 71 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 72 | @echo 73 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 74 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 75 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/transducer.qhcp" 76 | @echo "To view the help file:" 77 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/transducer.qhc" 78 | 79 | devhelp: 80 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 81 | @echo 82 | @echo "Build finished." 83 | @echo "To view the help file:" 84 | @echo "# mkdir -p $$HOME/.local/share/devhelp/transducer" 85 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/transducer" 86 | @echo "# devhelp" 87 | 88 | epub: 89 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 90 | @echo 91 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 92 | 93 | latex: 94 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 95 | @echo 96 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 97 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 98 | "(use \`make latexpdf' here to do that automatically)." 99 | 100 | latexpdf: 101 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 102 | @echo "Running LaTeX files through pdflatex..." 103 | make -C $(BUILDDIR)/latex all-pdf 104 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 105 | 106 | text: 107 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 108 | @echo 109 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 110 | 111 | man: 112 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 113 | @echo 114 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 115 | 116 | changes: 117 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 118 | @echo 119 | @echo "The overview file is in $(BUILDDIR)/changes." 120 | 121 | linkcheck: 122 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 123 | @echo 124 | @echo "Link check complete; look for any errors in the above output " \ 125 | "or in $(BUILDDIR)/linkcheck/output.txt." 126 | 127 | doctest: 128 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 129 | @echo "Testing of doctests in the sources finished, look at the " \ 130 | "results in $(BUILDDIR)/doctest/output.txt." 131 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source 10 | if NOT "%PAPER%" == "" ( 11 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 12 | ) 13 | 14 | if "%1" == "" goto help 15 | 16 | if "%1" == "help" ( 17 | :help 18 | echo.Please use `make ^` where ^ is one of 19 | echo. html to make standalone HTML files 20 | echo. dirhtml to make HTML files named index.html in directories 21 | echo. singlehtml to make a single large HTML file 22 | echo. pickle to make pickle files 23 | echo. json to make JSON files 24 | echo. htmlhelp to make HTML files and a HTML help project 25 | echo. qthelp to make HTML files and a qthelp project 26 | echo. devhelp to make HTML files and a Devhelp project 27 | echo. epub to make an epub 28 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 29 | echo. text to make text files 30 | echo. man to make manual pages 31 | echo. changes to make an overview over all changed/added/deprecated items 32 | echo. linkcheck to check all external links for integrity 33 | echo. doctest to run all doctests embedded in the documentation if enabled 34 | goto end 35 | ) 36 | 37 | if "%1" == "clean" ( 38 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 39 | del /q /s %BUILDDIR%\* 40 | goto end 41 | ) 42 | 43 | if "%1" == "html" ( 44 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 45 | if errorlevel 1 exit /b 1 46 | echo. 47 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 48 | goto end 49 | ) 50 | 51 | if "%1" == "dirhtml" ( 52 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 53 | if errorlevel 1 exit /b 1 54 | echo. 55 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 56 | goto end 57 | ) 58 | 59 | if "%1" == "singlehtml" ( 60 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 61 | if errorlevel 1 exit /b 1 62 | echo. 63 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 64 | goto end 65 | ) 66 | 67 | if "%1" == "pickle" ( 68 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 69 | if errorlevel 1 exit /b 1 70 | echo. 71 | echo.Build finished; now you can process the pickle files. 72 | goto end 73 | ) 74 | 75 | if "%1" == "json" ( 76 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 77 | if errorlevel 1 exit /b 1 78 | echo. 79 | echo.Build finished; now you can process the JSON files. 80 | goto end 81 | ) 82 | 83 | if "%1" == "htmlhelp" ( 84 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 85 | if errorlevel 1 exit /b 1 86 | echo. 87 | echo.Build finished; now you can run HTML Help Workshop with the ^ 88 | .hhp project file in %BUILDDIR%/htmlhelp. 89 | goto end 90 | ) 91 | 92 | if "%1" == "qthelp" ( 93 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 94 | if errorlevel 1 exit /b 1 95 | echo. 96 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 97 | .qhcp project file in %BUILDDIR%/qthelp, like this: 98 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\transducer.qhcp 99 | echo.To view the help file: 100 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\transducer.ghc 101 | goto end 102 | ) 103 | 104 | if "%1" == "devhelp" ( 105 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 106 | if errorlevel 1 exit /b 1 107 | echo. 108 | echo.Build finished. 109 | goto end 110 | ) 111 | 112 | if "%1" == "epub" ( 113 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 117 | goto end 118 | ) 119 | 120 | if "%1" == "latex" ( 121 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 122 | if errorlevel 1 exit /b 1 123 | echo. 124 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 125 | goto end 126 | ) 127 | 128 | if "%1" == "text" ( 129 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 130 | if errorlevel 1 exit /b 1 131 | echo. 132 | echo.Build finished. The text files are in %BUILDDIR%/text. 133 | goto end 134 | ) 135 | 136 | if "%1" == "man" ( 137 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 138 | if errorlevel 1 exit /b 1 139 | echo. 140 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 141 | goto end 142 | ) 143 | 144 | if "%1" == "changes" ( 145 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 146 | if errorlevel 1 exit /b 1 147 | echo. 148 | echo.The overview file is in %BUILDDIR%/changes. 149 | goto end 150 | ) 151 | 152 | if "%1" == "linkcheck" ( 153 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 154 | if errorlevel 1 exit /b 1 155 | echo. 156 | echo.Link check complete; look for any errors in the above output ^ 157 | or in %BUILDDIR%/linkcheck/output.txt. 158 | goto end 159 | ) 160 | 161 | if "%1" == "doctest" ( 162 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 163 | if errorlevel 1 exit /b 1 164 | echo. 165 | echo.Testing of doctests in the sources finished, look at the ^ 166 | results in %BUILDDIR%/doctest/output.txt. 167 | goto end 168 | ) 169 | 170 | :end 171 | -------------------------------------------------------------------------------- /test/test_sinks.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from io import StringIO 3 | 4 | from transducer.sinks import rprint, null_sink, CollectingSink, SingularSink 5 | 6 | 7 | class TestNullSink(unittest.TestCase): 8 | 9 | def test_sent_items_are_sunk(self): 10 | sink = null_sink() 11 | for i in range(100): 12 | sink.send(100) 13 | sink.close() 14 | 15 | def test_closed_sink_raises_stop_iteration(self): 16 | sink = null_sink() 17 | sink.close() 18 | with self.assertRaises(StopIteration): 19 | sink.send(42) 20 | 21 | 22 | class TestRPrint(unittest.TestCase): 23 | 24 | def test_sent_items_are_printed(self): 25 | with StringIO() as stream: 26 | sink = rprint(file=stream, flush=True) 27 | sink.send(10) 28 | sink.send(20) 29 | sink.send(30) 30 | result = stream.getvalue() 31 | self.assertEqual(result, "10\n20\n30") 32 | sink.close() 33 | 34 | def test_separators_are_printed(self): 35 | with StringIO() as stream: 36 | sink = rprint(sep=', ', file=stream, flush=True) 37 | sink.send(12) 38 | sink.send(24) 39 | sink.send(36) 40 | result = stream.getvalue() 41 | self.assertEqual(result, "12, 24, 36") 42 | sink.close() 43 | 44 | def test_end_terminator_is_printed(self): 45 | with StringIO() as stream: 46 | sink = rprint(end='END', file=stream, flush=True) 47 | sink.send(7) 48 | sink.send(14) 49 | sink.send(21) 50 | sink.close() 51 | result = stream.getvalue() 52 | self.assertEqual(result, "7\n14\n21END") 53 | 54 | def test_closed_sink_raises_stop_iteration(self): 55 | with StringIO() as stream: 56 | sink = rprint() 57 | sink.close() 58 | with self.assertRaises(StopIteration): 59 | sink.send("StopIteration should be raised") 60 | 61 | 62 | class TestCollectingSink(unittest.TestCase): 63 | 64 | def test_no_items_is_empty(self): 65 | collection = CollectingSink() 66 | self.assertEqual(len(collection), 0) 67 | 68 | def test_send_single_item_has_len_one(self): 69 | collection = CollectingSink() 70 | sink = collection() 71 | sink.send(42) 72 | self.assertEqual(len(collection), 1) 73 | 74 | def test_send_single_item_is_retrievable(self): 75 | collection = CollectingSink() 76 | sink = collection() 77 | sink.send(64) 78 | result = list(collection) 79 | self.assertListEqual(result, [64]) 80 | 81 | def test_multiple_items_are_retrievable(self): 82 | collection = CollectingSink() 83 | sink = collection() 84 | sink.send(64) 85 | sink.send(128) 86 | sink.send(256) 87 | result = list(collection) 88 | self.assertListEqual(result, [64, 128, 256]) 89 | 90 | def test_three_items_added_two_dequeued(self): 91 | collection = CollectingSink() 92 | sink = collection() 93 | sink.send(64) 94 | sink.send(128) 95 | sink.send(256) 96 | i = iter(collection) 97 | next(i) 98 | next(i) 99 | self.assertEqual(len(collection), 1) 100 | 101 | def test_three_items_added_four_dequeued_raises_stop_iteration(self): 102 | collection = CollectingSink() 103 | sink = collection() 104 | sink.send(64) 105 | sink.send(128) 106 | sink.send(256) 107 | i = iter(collection) 108 | next(i) 109 | next(i) 110 | next(i) 111 | with self.assertRaises(StopIteration): 112 | next(i) 113 | 114 | def test_send_items_to_multiple_sinks(self): 115 | collection = CollectingSink() 116 | sink1 = collection() 117 | sink2 = collection() 118 | sink1.send(64) 119 | sink2.send(128) 120 | sink1.send(256) 121 | sink2.send(512) 122 | result = list(collection) 123 | self.assertListEqual(result, [64, 128, 256, 512]) 124 | 125 | def test_send_items_then_clear_is_empty(self): 126 | collection = CollectingSink() 127 | sink = collection() 128 | sink.send(64) 129 | sink.send(128) 130 | sink.send(256) 131 | collection.clear() 132 | self.assertEqual(len(collection), 0) 133 | 134 | def test_closed_sink_raises_stop_iteration(self): 135 | collection = CollectingSink() 136 | sink = collection() 137 | sink.close() 138 | with self.assertRaises(StopIteration): 139 | sink.send(42) 140 | 141 | 142 | class TestSingularSink(unittest.TestCase): 143 | 144 | def test_no_items_raises_runtime_error(self): 145 | singular_sink = SingularSink() 146 | sink = singular_sink() 147 | with self.assertRaises(RuntimeError): 148 | _ = singular_sink.value 149 | 150 | def test_one_sent_item_can_be_retrieved(self): 151 | singular_sink = SingularSink() 152 | sink = singular_sink() 153 | sink.send(496) 154 | self.assertEqual(singular_sink.value, 496) 155 | 156 | def test_two_items_sent_raises_stop_iteration(self): 157 | singular_sink = SingularSink() 158 | sink = singular_sink() 159 | sink.send(342) 160 | with self.assertRaises(StopIteration): 161 | sink.send(124) 162 | 163 | def test_closed_sink_raises_stop_iteration(self): 164 | singular_sink = SingularSink() 165 | sink = singular_sink() 166 | sink.close() 167 | with self.assertRaises(StopIteration): 168 | sink.send(42) 169 | 170 | def test_zero_sent_items_has_no_value(self): 171 | singular_sink = SingularSink() 172 | sink = singular_sink() 173 | self.assertFalse(singular_sink.has_value) 174 | 175 | def test_one_sent_item_has_value(self): 176 | singular_sink = SingularSink() 177 | sink = singular_sink() 178 | sink.send(78) 179 | self.assertTrue(singular_sink.has_value) -------------------------------------------------------------------------------- /test/test_reducers.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from transducer._util import empty_iter 3 | from transducer.eager import transduce 4 | from transducer.infrastructure import Transducer 5 | from transducer.reducers import expecting_single, appending, conjoining, adding, sending, completing 6 | from transducer.sinks import CollectingSink, SingularSink 7 | from transducer.transducers import mapping 8 | 9 | 10 | class TestAppending(unittest.TestCase): 11 | 12 | def test_zero_items_returns_initial_empty_list(self): 13 | result = transduce(Transducer, 14 | appending(), 15 | empty_iter()) 16 | self.assertEqual(result, []) 17 | 18 | def test_two_items_returns_two_element_list(self): 19 | result = transduce(Transducer, 20 | appending(), 21 | (23, 78)) 22 | self.assertEqual(result, [23, 78]) 23 | 24 | def test_appending_to_immutable_sequence_raises_attribute_error(self): 25 | with self.assertRaises(AttributeError): 26 | transduce(Transducer, 27 | appending(), 28 | (23, 78), 29 | init=tuple()) 30 | 31 | 32 | class TestConjoining(unittest.TestCase): 33 | 34 | def test_zero_items_returns_initial_empty_tuple(self): 35 | result = transduce(Transducer, 36 | conjoining(), 37 | empty_iter()) 38 | self.assertEqual(result, tuple()) 39 | 40 | def test_two_items_returns_two_element_tuple(self): 41 | result = transduce(Transducer, 42 | conjoining(), 43 | [23, 78]) 44 | self.assertEqual(result, (23, 78)) 45 | 46 | def test_conjoining_to_non_sequence_raises_type_error(self): 47 | with self.assertRaises(TypeError): 48 | transduce(Transducer, 49 | conjoining(), 50 | (23, 78), 51 | init=set()) 52 | 53 | def test_conjoining_preserves_initial_sequence_type(self): 54 | result = transduce(Transducer, 55 | conjoining(), 56 | (23, 78), 57 | init=[]) 58 | self.assertEqual(result, [23, 78]) 59 | 60 | 61 | class TestAdding(unittest.TestCase): 62 | 63 | def test_zero_items_returns_initial_empty_set(self): 64 | result = transduce(Transducer, 65 | adding(), 66 | empty_iter()) 67 | self.assertEqual(result, set()) 68 | 69 | def test_two_items_returns_two_element_list(self): 70 | result = transduce(Transducer, 71 | adding(), 72 | [23, 78]) 73 | self.assertEqual(result, {23, 78}) 74 | 75 | def test_adding_to_non_set_raises_attribute_error(self): 76 | with self.assertRaises(AttributeError): 77 | transduce(Transducer, 78 | adding(), 79 | (23, 78), 80 | init=tuple()) 81 | 82 | 83 | class TestExpectingSingle(unittest.TestCase): 84 | 85 | def test_too_few_items(self): 86 | with self.assertRaises(RuntimeError): 87 | transduce(mapping(lambda x: x*x), 88 | expecting_single(), 89 | [1, 2]) 90 | 91 | def test_exactly_one_item(self): 92 | result = transduce(mapping(lambda x: x*x), 93 | expecting_single(), 94 | [42]) 95 | self.assertEqual(result, 1764) 96 | 97 | def test_too_many_items(self): 98 | with self.assertRaises(RuntimeError): 99 | transduce(mapping(lambda x: x*x), 100 | expecting_single(), 101 | []) 102 | 103 | 104 | class TestSending(unittest.TestCase): 105 | 106 | def test_zero_items_returns_initial_empty_collection(self): 107 | collection = CollectingSink() 108 | 109 | transduce(Transducer, 110 | sending(), 111 | empty_iter(), 112 | init=collection()) 113 | 114 | self.assertEqual(len(collection), 0) 115 | 116 | def test_two_items_returns_two_element_list(self): 117 | collection = CollectingSink() 118 | 119 | transduce(Transducer, 120 | sending(), 121 | [23, 78], 122 | init=collection()) 123 | 124 | self.assertEqual(list(collection), [23, 78]) 125 | 126 | def test_sending_to_non_sink_raises_attribute_error(self): 127 | with self.assertRaises(AttributeError): 128 | transduce(Transducer, 129 | sending(), 130 | (23, 78), 131 | init=set()) 132 | 133 | def test_two_items_causes_completion(self): 134 | singular_sink = SingularSink() 135 | 136 | transduce(Transducer, 137 | sending(), 138 | [23, 78], 139 | init=singular_sink()) 140 | 141 | self.assertTrue(singular_sink.has_value) 142 | 143 | 144 | class TestCompleting(unittest.TestCase): 145 | 146 | def test_completing_with_summing_zero_items_returns_identity(self): 147 | 148 | def add(a, b): 149 | return a + b 150 | 151 | summing = completing(add, identity=0) 152 | 153 | result = transduce(Transducer, 154 | summing, 155 | []) 156 | self.assertEqual(result, 0) 157 | 158 | def test_completing_with_summing_four_items(self): 159 | 160 | def add(a, b): 161 | return a + b 162 | 163 | summing = completing(add, identity=0) 164 | 165 | result = transduce(Transducer, 166 | summing, 167 | [4, 2, 1, 9]) 168 | self.assertEqual(result, 16) 169 | 170 | def test_completing_with_multiplying_zero_items_returns_identity(self): 171 | 172 | def multiply(a, b): 173 | return a * b 174 | 175 | multiplying = completing(multiply, identity=1) 176 | 177 | result = transduce(Transducer, 178 | multiplying, 179 | []) 180 | self.assertEqual(result, 1) 181 | 182 | def test_completing_with_multiplying_four_items(self): 183 | 184 | def multiply(a, b): 185 | return a * b 186 | 187 | multiplying = completing(multiply, identity=1) 188 | 189 | result = transduce(Transducer, 190 | multiplying, 191 | [4, 2, 1, 9]) 192 | self.assertEqual(result, 72) 193 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # transducer documentation build configuration file, created by 4 | # sphinx-quickstart on Thu Feb 17 19:12:20 2011. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | sys.path.insert(0, os.path.abspath(os.path.join('..', '..'))) 20 | 21 | import transducer 22 | 23 | # on_rtd is whether we are on readthedocs.org 24 | import os 25 | on_rtd = os.environ.get('READTHEDOCS', None) == 'True' 26 | 27 | if not on_rtd: # only import and set the theme if we're building docs locally 28 | import sphinx_rtd_theme 29 | html_theme = 'sphinx_rtd_theme' 30 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 31 | 32 | # otherwise, readthedocs.org uses their theme by default, so no need to specify it 33 | 34 | # -- General configuration ----------------------------------------------------- 35 | 36 | # If your documentation needs a minimal Sphinx version, state it here. 37 | #needs_sphinx = '1.0' 38 | 39 | # Add any Sphinx extension module names here, as strings. They can be extensions 40 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 41 | 42 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'cartouche'] 43 | 44 | # Add any paths that contain templates here, relative to this directory. 45 | templates_path = ['_templates'] 46 | 47 | # The suffix of source filenames. 48 | source_suffix = '.rst' 49 | 50 | # The encoding of source files. 51 | #source_encoding = 'utf-8-sig' 52 | 53 | # The master toctree document. 54 | master_doc = 'index' 55 | 56 | # General information about the project. 57 | project = 'transducer' 58 | copyright = '2015, Sixty North' 59 | 60 | # The version info for the project you're documenting, acts as replacement for 61 | # |version| and |release|, also used in various other places throughout the 62 | # built documents. 63 | # 64 | # The short X.Y version. 65 | 66 | version = transducer.__version__ 67 | # The full version, including alpha/beta/rc tags. 68 | release = transducer.__version__ 69 | 70 | # The language for content autogenerated by Sphinx. Refer to documentation 71 | # for a list of supported languages. 72 | #language = None 73 | 74 | # There are two options for replacing |today|: either, you set today to some 75 | # non-false value, then it is used: 76 | #today = '' 77 | # Else, today_fmt is used as the format for a strftime call. 78 | #today_fmt = '%B %d, %Y' 79 | 80 | # List of patterns, relative to source directory, that match files and 81 | # directories to ignore when looking for source files. 82 | exclude_patterns = [] 83 | 84 | # The reST default role (used for this markup: `text`) to use for all documents. 85 | #default_role = None 86 | 87 | # If true, '()' will be appended to :func: etc. cross-reference text. 88 | #add_function_parentheses = True 89 | 90 | # If true, the current module name will be prepended to all description 91 | # unit titles (such as .. function::). 92 | #add_module_names = True 93 | 94 | # If true, sectionauthor and moduleauthor directives will be shown in the 95 | # output. They are ignored by default. 96 | #show_authors = False 97 | 98 | # The name of the Pygments (syntax highlighting) style to use. 99 | pygments_style = 'sphinx' 100 | 101 | # A list of ignored prefixes for module index sorting. 102 | #modindex_common_prefix = [] 103 | 104 | 105 | # -- Options for HTML output --------------------------------------------------- 106 | 107 | # The theme to use for HTML and HTML Help pages. See the documentation for 108 | # a list of builtin themes. 109 | 110 | # Theme options are theme-specific and customize the look and feel of a theme 111 | # further. For a list of options available for each theme, see the 112 | # documentation. 113 | #html_theme_options = {} 114 | 115 | # Add any paths that contain custom themes here, relative to this directory. 116 | #html_theme_path = [] 117 | 118 | # The name for this set of Sphinx documents. If None, it defaults to 119 | # " v documentation". 120 | #html_title = None 121 | 122 | # A shorter title for the navigation bar. Default is the same as html_title. 123 | #html_short_title = None 124 | 125 | # The name of an image file (relative to this directory) to place at the top 126 | # of the sidebar. 127 | html_logo = 'images/transducer_logo_150.png' 128 | 129 | # The name of an image file (within the static path) to use as favicon of the 130 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 131 | # pixels large. 132 | #html_favicon = None 133 | 134 | # Add any paths that contain custom static files (such as style sheets) here, 135 | # relative to this directory. They are copied after the builtin static files, 136 | # so a file named "default.css" will overwrite the builtin "default.css". 137 | html_static_path = ['_static'] 138 | 139 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 140 | # using the given strftime format. 141 | #html_last_updated_fmt = '%b %d, %Y' 142 | 143 | # If true, SmartyPants will be used to convert quotes and dashes to 144 | # typographically correct entities. 145 | #html_use_smartypants = True 146 | 147 | # Custom sidebar templates, maps document names to template names. 148 | #html_sidebars = {} 149 | 150 | # Additional templates that should be rendered to pages, maps page names to 151 | # template names. 152 | #html_additional_pages = {} 153 | 154 | # If false, no module index is generated. 155 | #html_domain_indices = True 156 | 157 | # If false, no index is generated. 158 | #html_use_index = True 159 | 160 | # If true, the index is split into individual pages for each letter. 161 | #html_split_index = False 162 | 163 | # If true, links to the reST sources are added to the pages. 164 | #html_show_sourcelink = True 165 | 166 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 167 | #html_show_sphinx = True 168 | 169 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 170 | #html_show_copyright = True 171 | 172 | # If true, an OpenSearch description file will be output, and all pages will 173 | # contain a tag referring to it. The value of this option must be the 174 | # base URL from which the finished HTML is served. 175 | #html_use_opensearch = '' 176 | 177 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 178 | #html_file_suffix = None 179 | 180 | # Output file base name for HTML help builder. 181 | htmlhelp_basename = 'transducerdoc' 182 | 183 | 184 | # -- Options for LaTeX output -------------------------------------------------- 185 | 186 | # The paper size ('letter' or 'a4'). 187 | #latex_paper_size = 'letter' 188 | 189 | # The font size ('10pt', '11pt' or '12pt'). 190 | #latex_font_size = '10pt' 191 | 192 | # Grouping the document tree into LaTeX files. List of tuples 193 | # (source start file, target name, title, author, documentclass [howto/manual]). 194 | latex_documents = [ 195 | ('index', 'transducer.tex', 'transducer Documentation', 196 | 'Sixty North', 'manual'), 197 | ] 198 | 199 | # The name of an image file (relative to this directory) to place at the top of 200 | # the title page. 201 | #latex_logo = None 202 | 203 | # For "manual" documents, if this is true, then toplevel headings are parts, 204 | # not chapters. 205 | #latex_use_parts = False 206 | 207 | # If true, show page references after internal links. 208 | #latex_show_pagerefs = False 209 | 210 | # If true, show URL addresses after external links. 211 | #latex_show_urls = False 212 | 213 | # Additional stuff for the LaTeX preamble. 214 | #latex_preamble = '' 215 | 216 | # Documents to append as an appendix to all manuals. 217 | #latex_appendices = [] 218 | 219 | # If false, no module index is generated. 220 | #latex_domain_indices = True 221 | 222 | 223 | # -- Options for manual page output -------------------------------------------- 224 | 225 | # One entry per manual page. List of tuples 226 | # (source start file, name, description, authors, manual section). 227 | man_pages = [ 228 | ('index', 'transducer', 'transducer Documentation', 229 | ['Sixty North'], 1) 230 | ] 231 | 232 | add_module_names = False -------------------------------------------------------------------------------- /test/test_eager.py: -------------------------------------------------------------------------------- 1 | from collections import deque 2 | import operator 3 | import unittest 4 | from transducer.eager import transduce 5 | from transducer.functional import compose 6 | from transducer.infrastructure import Transducer 7 | from transducer.reducers import appending, expecting_single, conjoining, adding 8 | from transducer.transducers import (mapping, filtering, reducing, enumerating, first, last, 9 | reversing, ordering, counting, scanning, taking, dropping_while, distinct, 10 | taking_while, dropping, element_at, mapcatting, pairwise, batching, windowing, 11 | repeating) 12 | 13 | 14 | class TestSingleTransducers(unittest.TestCase): 15 | 16 | def test_identity(self): 17 | result = transduce(transducer=Transducer, 18 | reducer=appending(), 19 | iterable=range(5)) 20 | self.assertListEqual(result, [0, 1, 2, 3, 4]) 21 | 22 | def test_mapping(self): 23 | result = transduce(transducer=mapping(lambda x: x*x), 24 | reducer=appending(), 25 | iterable=range(5)) 26 | self.assertListEqual(result, [0, 1, 4, 9, 16]) 27 | 28 | def test_filtering(self): 29 | result = transduce(transducer=filtering(lambda w: 'x' in w), 30 | reducer=appending(), 31 | iterable='socks in box fox on clocks'.split()) 32 | self.assertListEqual(result, ['box', 'fox']) 33 | 34 | def test_reducing(self): 35 | result = transduce(transducer=reducing(operator.add), 36 | reducer=expecting_single(), 37 | iterable=range(10)) 38 | self.assertEqual(result, 45) 39 | 40 | def test_reducing_with_init(self): 41 | result = transduce(transducer=reducing(operator.add, 10), 42 | reducer=expecting_single(), 43 | iterable=range(10)) 44 | self.assertEqual(result, 55) 45 | 46 | def test_scanning(self): 47 | result = transduce(transducer=scanning(operator.add), 48 | reducer=appending(), 49 | iterable=range(5)) 50 | self.assertListEqual(result, [0, 1, 3, 6, 10]) 51 | 52 | def test_scanning_with_init(self): 53 | result = transduce(transducer=scanning(operator.add, 3), 54 | reducer=appending(), 55 | iterable=range(5)) 56 | self.assertListEqual(result, [3, 4, 6, 9, 13]) 57 | 58 | def test_enumerating(self): 59 | result = transduce(transducer=enumerating(), 60 | reducer=appending(), 61 | iterable=[2, 4, 6, 8, 10]) 62 | self.assertListEqual(result, [(0, 2), (1, 4), (2, 6), (3, 8), (4, 10)]) 63 | 64 | def test_enumerating_with_start(self): 65 | result = transduce(transducer=enumerating(start=3), 66 | reducer=appending(), 67 | iterable=[2, 4, 6, 8, 10]) 68 | self.assertListEqual(result, [(3, 2), (4, 4), (5, 6), (6, 8), (7, 10)]) 69 | 70 | def test_mapcatting(self): 71 | result = transduce(transducer=mapcatting(list), 72 | reducer=appending(), 73 | iterable=['new', 'found', 'land']) 74 | self.assertListEqual(result, list("newfoundland")) 75 | 76 | def test_taking(self): 77 | result = transduce(transducer=taking(3), 78 | reducer=appending(), 79 | iterable=[2, 4, 5, 8, 10]) 80 | self.assertListEqual(result, [2, 4, 5]) 81 | 82 | def test_taking_validation(self): 83 | with self.assertRaises(ValueError): 84 | transduce(transducer=taking(-3), 85 | reducer=appending(), 86 | iterable=[2, 4, 5, 8, 10]) 87 | 88 | def test_taking_while(self): 89 | result = transduce(transducer=taking_while(lambda x: x < 6), 90 | reducer=appending(), 91 | iterable=[2, 4, 5, 8, 10]) 92 | self.assertListEqual(result, [2, 4, 5]) 93 | 94 | def test_dropping(self): 95 | result = transduce(transducer=dropping(3), 96 | reducer=appending(), 97 | iterable=[2, 4, 5, 8, 10]) 98 | self.assertListEqual(result, [8, 10]) 99 | 100 | def test_dropping_validation(self): 101 | with self.assertRaises(ValueError): 102 | transduce(transducer=dropping(-3), 103 | reducer=appending(), 104 | iterable=[2, 4, 5, 8, 10]) 105 | 106 | def test_dropping_while(self): 107 | result = transduce(transducer=dropping_while(lambda x: x < 6), 108 | reducer=appending(), 109 | iterable=[2, 4, 5, 8, 10]) 110 | self.assertListEqual(result, [8, 10]) 111 | 112 | def test_distinct(self): 113 | result = transduce(transducer=distinct(), 114 | reducer=appending(), 115 | iterable=[1, 1, 3, 5, 5, 2, 1, 2]) 116 | self.assertListEqual(result, [1, 3, 5, 2]) 117 | 118 | def test_pairwise_at_least_two(self): 119 | result = transduce(transducer=pairwise(), 120 | reducer=appending(), 121 | iterable=[1, 3, 5, 7, 2, 1, 9]) 122 | self.assertListEqual(result, [(1, 3), (3, 5), (5, 7), (7, 2), (2, 1), (1, 9)]) 123 | 124 | def test_pairwise_single(self): 125 | """A single item fed into pairwise is discarded.""" 126 | result = transduce(transducer=pairwise(), 127 | reducer=appending(), 128 | iterable=[42]) 129 | self.assertListEqual(result, []) 130 | 131 | def test_batching_exact(self): 132 | result = transduce(transducer=batching(3), 133 | reducer=appending(), 134 | iterable=[42, 12, 45, 9, 18, 3, 34, 13, 12]) 135 | self.assertListEqual(result, [[42, 12, 45], [9, 18, 3], [34, 13, 12]]) 136 | 137 | def test_batching_inexact_1(self): 138 | result = transduce(transducer=batching(3), 139 | reducer=appending(), 140 | iterable=[42, 12, 45, 9, 18, 3, 34]) 141 | self.assertListEqual(result, [[42, 12, 45], [9, 18, 3], [34]]) 142 | 143 | def test_batching_inexact_2(self): 144 | result = transduce(transducer=batching(3), 145 | reducer=appending(), 146 | iterable=[42, 12, 45, 9, 18, 3, 34, 13]) 147 | self.assertListEqual(result, [[42, 12, 45], [9, 18, 3], [34, 13]]) 148 | 149 | def test_batching_validation(self): 150 | with self.assertRaises(ValueError): 151 | transduce(transducer=batching(0), 152 | reducer=appending(), 153 | iterable=[42, 12, 45, 9, 18, 3, 34, 13]) 154 | 155 | def test_windowing_no_padding(self): 156 | result = transduce(transducer=windowing(3, window_type=list), 157 | reducer=appending(), 158 | iterable=[42, 12, 45, 9, 18, 3, 34, 13]) 159 | self.assertListEqual(result, 160 | [[42], 161 | [42, 12], 162 | [42, 12, 45], 163 | [12, 45, 9], 164 | [45, 9, 18], 165 | [9, 18, 3], 166 | [18, 3, 34], 167 | [3, 34, 13], 168 | [34, 13], 169 | [13]]) 170 | 171 | def test_windowing_padding(self): 172 | result = transduce(transducer=windowing(3, padding=0, window_type=list), 173 | reducer=appending(), 174 | iterable=[42, 12, 45, 9, 18, 3, 34, 13]) 175 | self.assertListEqual(result, 176 | [[0, 0, 42], 177 | [0, 42, 12], 178 | [42, 12, 45], 179 | [12, 45, 9], 180 | [45, 9, 18], 181 | [9, 18, 3], 182 | [18, 3, 34], 183 | [3, 34, 13], 184 | [34, 13, 0], 185 | [13, 0, 0]]) 186 | 187 | def test_windowing_validation(self): 188 | with self.assertRaises(ValueError): 189 | transduce(transducer=windowing(0), 190 | reducer=appending(), 191 | iterable=[42, 12, 45, 9, 18, 3, 34, 13]) 192 | 193 | def test_element_at(self): 194 | result = transduce(transducer=element_at(3), 195 | reducer=expecting_single(), 196 | iterable=[1, 3, 5, 7, 9]) 197 | self.assertEqual(result, 7) 198 | 199 | def test_element_at_validation(self): 200 | with self.assertRaises(IndexError): 201 | transduce(transducer=element_at(-1), 202 | reducer=expecting_single(), 203 | iterable=[1, 3, 5, 7, 9]) 204 | 205 | def test_element_at_too_short(self): 206 | with self.assertRaises(IndexError): 207 | transduce(transducer=element_at(3), 208 | reducer=expecting_single(), 209 | iterable=[1, 3, 5]) 210 | 211 | def test_repeating(self): 212 | result = transduce(transducer=repeating(3), 213 | reducer=appending(), 214 | iterable=[1, 3, 5]) 215 | self.assertListEqual(result, [1, 1, 1, 3, 3, 3, 5, 5, 5]) 216 | 217 | def test_repeating_zero(self): 218 | result = transduce(transducer=repeating(0), 219 | reducer=appending(), 220 | iterable=[1, 3, 5]) 221 | self.assertListEqual(result, []) 222 | 223 | def test_repeating_validation(self): 224 | with self.assertRaises(ValueError): 225 | transduce(transducer=repeating(-1), 226 | reducer=appending(), 227 | iterable=[1, 3, 5]) 228 | 229 | def test_first(self): 230 | result = transduce(transducer=first(), 231 | reducer=expecting_single(), 232 | iterable=[2, 4, 6, 8, 10]) 233 | self.assertEqual(result, 2) 234 | 235 | def test_first_with_predicate(self): 236 | result = transduce(transducer=first(lambda x: x > 5), 237 | reducer=expecting_single(), 238 | iterable=[2, 4, 6, 8, 10]) 239 | self.assertEqual(result, 6) 240 | 241 | def test_last(self): 242 | result = transduce(transducer=last(), 243 | reducer=expecting_single(), 244 | iterable=[2, 4, 6, 8, 10]) 245 | self.assertEqual(result, 10) 246 | 247 | def test_last_with_predicate(self): 248 | result = transduce(transducer=last(lambda x: x < 7), 249 | reducer=expecting_single(), 250 | iterable=[2, 4, 6, 8, 10]) 251 | self.assertEqual(result, 6) 252 | 253 | def test_reversing(self): 254 | result = transduce(transducer=reversing(), 255 | reducer=appending(), 256 | iterable=[2, 4, 6, 8, 10]) 257 | self.assertSequenceEqual(result, [10, 8, 6, 4, 2]) 258 | 259 | def test_reversing_preserves_mutable_sequence_type(self): 260 | result = transduce(transducer=reversing(), 261 | reducer=appending(), 262 | iterable=[2, 4, 6, 8, 10]) 263 | self.assertIsInstance(result, list) 264 | self.assertSequenceEqual(result, [10, 8, 6, 4, 2]) 265 | 266 | def test_ordering(self): 267 | result = transduce(transducer=ordering(), 268 | reducer=appending(), 269 | iterable=[4, 2, 6, 10, 8]) 270 | self.assertSequenceEqual(result, [2, 4, 6, 8, 10]) 271 | 272 | def test_ordering_preserves_mutable_sequence_type(self): 273 | result = transduce(transducer=ordering(), 274 | reducer=appending(), 275 | iterable=[4, 2, 6, 10, 8], 276 | init=deque()) 277 | self.assertIsInstance(result, deque) 278 | self.assertSequenceEqual(result, deque([2, 4, 6, 8, 10])) 279 | 280 | def test_ordering_preserves_immutable_sequence_type(self): 281 | result = transduce(transducer=ordering(), 282 | reducer=conjoining(), 283 | iterable=[4, 2, 6, 10, 8]) 284 | self.assertIsInstance(result, tuple) 285 | self.assertSequenceEqual(result, (2, 4, 6, 8, 10)) 286 | 287 | def test_ordering_reverse(self): 288 | result = transduce(transducer=ordering(reverse=True), 289 | reducer=appending(), 290 | iterable=[4, 2, 6, 10, 8]) 291 | self.assertSequenceEqual(result, [10, 8, 6, 4, 2]) 292 | 293 | def test_ordering_with_key(self): 294 | result = transduce(transducer=ordering(key=lambda x: len(x)), 295 | reducer=appending(), 296 | iterable="The quick brown fox jumped".split()) 297 | self.assertSequenceEqual(result, ['The', 'fox', 'quick', 'brown', 'jumped']) 298 | 299 | def test_ordering_reverse_with_key(self): 300 | result = transduce(transducer=ordering(key=lambda x: len(x), reverse=True), 301 | reducer=appending(), 302 | iterable="The quick brown fox jumped".split()) 303 | self.assertSequenceEqual(result, ['jumped', 'quick', 'brown', 'The', 'fox']) 304 | 305 | def test_counting(self): 306 | result = transduce(transducer=counting(), 307 | reducer=expecting_single(), 308 | iterable="The quick brown fox jumped".split()) 309 | self.assertEqual(result, 5) 310 | 311 | def test_counting_with_predicate(self): 312 | result = transduce(transducer=counting(lambda w: 'o' in w), 313 | reducer=expecting_single(), 314 | iterable="The quick brown fox jumped".split()) 315 | self.assertEqual(result, 2) 316 | 317 | def test_mutable_inits(self): 318 | """Tests that the same mutable init object isn't shared across invocations.""" 319 | result = transduce(transducer=mapping(lambda x: x), reducer=appending(), iterable=range(3)) 320 | self.assertListEqual(result, [0, 1, 2]) 321 | result = transduce(transducer=mapping(lambda x: x), reducer=appending(), iterable=range(3)) 322 | self.assertListEqual(result, [0, 1, 2]) 323 | 324 | def test_adding_reducer(self): 325 | result = transduce( 326 | transducer=mapping(lambda x: x * x), 327 | reducer=adding(), 328 | iterable=list(range(3)) * 2) 329 | self.assertListEqual(list(result), [0, 1, 4]) 330 | 331 | 332 | class TestComposedTransducers(unittest.TestCase): 333 | 334 | def test_chained_transducers(self): 335 | result = transduce(transducer=compose( 336 | mapping(lambda x: x*x), 337 | filtering(lambda x: x % 5 != 0), 338 | taking(6), 339 | dropping_while(lambda x: x < 15), 340 | distinct()), 341 | reducer=appending(), 342 | iterable=range(20)) 343 | self.assertSequenceEqual(result, [16, 36, 49]) 344 | 345 | 346 | if __name__ == '__main__': 347 | unittest.main() 348 | -------------------------------------------------------------------------------- /transducer/transducers.py: -------------------------------------------------------------------------------- 1 | """Functions for creating transducers. 2 | 3 | The functions in this module return transducers. 4 | """ 5 | from collections import deque 6 | from functools import reduce 7 | 8 | from transducer._util import UNSET 9 | from transducer.functional import true 10 | from transducer.infrastructure import Reduced, Transducer 11 | 12 | 13 | # Functions for creating transducers, which are themselves 14 | # functions which transform one reducer to another 15 | 16 | # --------------------------------------------------------------------- 17 | 18 | class Mapping(Transducer): 19 | 20 | def __init__(self, reducer, transform): 21 | super().__init__(reducer) 22 | self._transform = transform 23 | 24 | def step(self, result, item): 25 | return self._reducer(result, self._transform(item)) 26 | 27 | 28 | def mapping(transform): 29 | """Create a mapping transducer with the given transform. 30 | 31 | Args: 32 | transform: A single-argument function which will be applied to 33 | each input element to produce the corresponding output 34 | element. 35 | 36 | Returns: A mapping transducer: A single argument function which, 37 | when passed a reducing function, returns a new reducing function 38 | which applies the specified mapping transform function *before* 39 | delegating to the original reducer. 40 | """ 41 | 42 | def mapping_transducer(reducer): 43 | return Mapping(reducer, transform) 44 | 45 | return mapping_transducer 46 | 47 | # --------------------------------------------------------------------- 48 | 49 | 50 | class Filtering(Transducer): 51 | 52 | def __init__(self, reducer, predicate): 53 | super().__init__(reducer) 54 | self._predicate = predicate 55 | 56 | def step(self, result, item): 57 | return self._reducer(result, item) if self._predicate(item) else result 58 | 59 | 60 | def filtering(predicate): 61 | """Create a filtering transducer with the given predicate. 62 | 63 | Args: 64 | predicate: A single-argument function which will be used to 65 | test each element and which must return True or False. Only 66 | those elements for which this function returns True will be 67 | retained. 68 | 69 | Returns: A filtering transducer: A single argument function which, 70 | when passed a reducing function, returns a new reducing function 71 | which passes only those items in the input stream which match 72 | satisfy the predicate to the original reducer. 73 | """ 74 | 75 | def filtering_transducer(reducer): 76 | return Filtering(reducer, predicate) 77 | 78 | return filtering_transducer 79 | 80 | 81 | # --------------------------------------------------------------------- 82 | 83 | class Reducing(Transducer): 84 | 85 | def __init__(self, reducer, reducer2, init=UNSET): 86 | super().__init__(reducer) 87 | self._reducer2 = reducer2 88 | self._accumulator = init # TODO: Should we try to call reducer2.initial() here? 89 | 90 | def step(self, result, item): 91 | self._accumulator = item if self._accumulator is UNSET else self._reducer2(self._accumulator, item) 92 | return result 93 | 94 | def complete(self, result): 95 | result = self._reducer.step(result, self._accumulator) 96 | return self._reducer.complete(result) 97 | 98 | 99 | def reducing(reducer, init=UNSET): 100 | """Create a reducing transducer with the given reducer. 101 | 102 | Args: 103 | reducer: A two-argument function which will be used to combine the 104 | partial cumulative result in the first argument with the next 105 | item from the input stream in the second argument. 106 | 107 | Returns: A reducing transducer: A single argument function which, 108 | when passed a reducing function, returns a new reducing function 109 | which entirely reduces the input stream using 'reducer' before 110 | passing the result to the reducing function passed to the 111 | transducer. 112 | """ 113 | 114 | reducer2 = reducer 115 | 116 | def reducing_transducer(reducer): 117 | return Reducing(reducer, reducer2, init) 118 | 119 | return reducing_transducer 120 | 121 | # --------------------------------------------------------------------- 122 | 123 | 124 | class Scanning(Transducer): 125 | 126 | def __init__(self, reducer, reducer2, init=UNSET): 127 | super().__init__(reducer) 128 | self._reducer2 = reducer2 129 | self._accumulator = init # TODO: Should we try to call reducer2.initial() here? 130 | 131 | def step(self, result, item): 132 | self._accumulator = item if self._accumulator is UNSET else self._reducer2(self._accumulator, item) 133 | return self._reducer.step(result, self._accumulator) 134 | 135 | 136 | def scanning(reducer, init=UNSET): 137 | """Create a scanning reducer.""" 138 | 139 | reducer2 = reducer 140 | 141 | def scanning_transducer(reducer): 142 | return Scanning(reducer, reducer2, init) 143 | 144 | return scanning_transducer 145 | 146 | 147 | # --------------------------------------------------------------------- 148 | 149 | 150 | class Enumerating(Transducer): 151 | 152 | def __init__(self, reducer, start): 153 | super().__init__(reducer) 154 | self._counter = start 155 | 156 | def step(self, result, item): 157 | index = self._counter 158 | self._counter += 1 159 | return self._reducer(result, (index, item)) 160 | 161 | 162 | def enumerating(start=0): 163 | """Create a transducer which enumerates items.""" 164 | 165 | def enumerating_transducer(reducer): 166 | return Enumerating(reducer, start) 167 | 168 | return enumerating_transducer 169 | 170 | # --------------------------------------------------------------------- 171 | 172 | 173 | class Mapcatting(Transducer): 174 | 175 | def __init__(self, reducer, transform): 176 | super().__init__(reducer) 177 | self._transform = transform 178 | 179 | def step(self, result, item): 180 | return reduce(self._reducer, self._transform(item), result) 181 | 182 | 183 | def mapcatting(transform): 184 | """Create a transducer which transforms items and concatenates the results""" 185 | 186 | def mapcatting_transducer(reducer): 187 | return Mapcatting(reducer, transform) 188 | 189 | return mapcatting_transducer 190 | 191 | # --------------------------------------------------------------------- 192 | 193 | 194 | class Taking(Transducer): 195 | 196 | def __init__(self, reducer, n): 197 | super().__init__(reducer) 198 | self._counter = 0 199 | self._n = n 200 | 201 | def step(self, result, item): 202 | self._counter += 1 203 | result = self._reducer(result, item) 204 | return Reduced(result) if self._counter >= self._n else result 205 | 206 | 207 | def taking(n): 208 | """Create a transducer which takes the first n items""" 209 | 210 | if n < 0: 211 | raise ValueError("Cannot take fewer than zero ({}) items".format(n)) 212 | 213 | def taking_transducer(reducer): 214 | return Taking(reducer, n) 215 | 216 | return taking_transducer 217 | 218 | # --------------------------------------------------------------------- 219 | 220 | 221 | class TakingWhile(Transducer): 222 | 223 | def __init__(self, reducer, predicate): 224 | super().__init__(reducer) 225 | self._predicate = predicate 226 | 227 | def step(self, result, item): 228 | return self._reducer(result, item) if self._predicate(item) else Reduced(result) 229 | 230 | 231 | def taking_while(predicate): 232 | """Create a transducer which takes leading items while they satisfy a predicate.""" 233 | 234 | def taking_while_transducer(reducer): 235 | return TakingWhile(reducer, predicate) 236 | 237 | return taking_while_transducer 238 | 239 | # --------------------------------------------------------------------- 240 | 241 | 242 | class Dropping(Transducer): 243 | 244 | def __init__(self, reducer, n): 245 | super().__init__(reducer) 246 | self._counter = 0 247 | self._n = n 248 | 249 | def step(self, result, item): 250 | result = result if self._counter < self._n else self._reducer(result, item) 251 | self._counter += 1 252 | return result 253 | 254 | 255 | def dropping(n): 256 | """Create a transducer which drops the first n items""" 257 | 258 | if n < 0: 259 | raise ValueError("Cannot drop fewer than zero ({}) items".format(n)) 260 | 261 | def dropping_transducer(reducer): 262 | return Dropping(reducer, n) 263 | 264 | return dropping_transducer 265 | 266 | 267 | class DroppingWhile(Transducer): 268 | 269 | def __init__(self, reducer, predicate): 270 | super().__init__(reducer) 271 | self._predicate = predicate 272 | self._dropping = True 273 | 274 | def step(self, result, item): 275 | self._dropping = self._dropping and self._predicate(item) 276 | return result if self._dropping else self._reducer(result, item) 277 | 278 | 279 | def dropping_while(predicate): 280 | """Create a transducer which drops leading items while a predicate holds.""" 281 | 282 | def dropping_while_transducer(reducer): 283 | return DroppingWhile(reducer, predicate) 284 | 285 | return dropping_while_transducer 286 | 287 | # --------------------------------------------------------------------- 288 | 289 | 290 | class Distinct(Transducer): 291 | 292 | def __init__(self, reducer): 293 | super().__init__(reducer) 294 | self._seen = set() 295 | 296 | def step(self, result, item): 297 | if item not in self._seen: 298 | self._seen.add(item) 299 | return self._reducer(result, item) 300 | return result 301 | 302 | 303 | def distinct(): 304 | """Create a transducer which filters distinct items""" 305 | 306 | def distinct_transducer(reducer): 307 | return Distinct(reducer) 308 | 309 | return distinct_transducer 310 | 311 | # --------------------------------------------------------------------- 312 | 313 | 314 | class Pairwise(Transducer): 315 | 316 | def __init__(self, reducer): 317 | super().__init__(reducer) 318 | self._previous_item = UNSET 319 | 320 | def step(self, result, item): 321 | if self._previous_item is UNSET: 322 | self._previous_item = item 323 | return result 324 | pair = (self._previous_item, item) 325 | self._previous_item = item 326 | return self._reducer.step(result, pair) 327 | 328 | 329 | def pairwise(): 330 | """Create a transducer which produces successive pairs""" 331 | 332 | def pairwise_transducer(reducer): 333 | return Pairwise(reducer) 334 | 335 | return pairwise_transducer 336 | 337 | # --------------------------------------------------------------------- 338 | 339 | 340 | class Batching(Transducer): 341 | 342 | def __init__(self, reducer, size): 343 | super().__init__(reducer) 344 | self._size = size 345 | self._pending = [] 346 | 347 | def step(self, result, item): 348 | self._pending.append(item) 349 | if len(self._pending) == self._size: 350 | batch = self._pending 351 | self._pending = [] 352 | return self._reducer(result, batch) 353 | return result 354 | 355 | def complete(self, result): 356 | r = self._reducer.step(result, self._pending) if len(self._pending) > 0 else result 357 | return self._reducer.complete(r) 358 | 359 | 360 | def batching(size): 361 | """Create a transducer which produces non-overlapping batches.""" 362 | 363 | if size < 1: 364 | raise ValueError("batching() size must be at least 1") 365 | 366 | def batching_transducer(reducer): 367 | return Batching(reducer, size) 368 | 369 | return batching_transducer 370 | 371 | # --------------------------------------------------------------------- 372 | 373 | 374 | class Windowing(Transducer): 375 | 376 | def __init__(self, reducer, size, padding, window_type): 377 | super().__init__(reducer) 378 | self._size = size 379 | self._padding = padding 380 | self._window = deque(maxlen=size) if padding is UNSET else deque([padding] * size, maxlen=size) 381 | self._window_type = window_type 382 | 383 | def step(self, result, item): 384 | self._window.append(item) 385 | return self._reducer.step(result, self._window_type(self._window)) 386 | 387 | def complete(self, result): 388 | if self._padding is not UNSET: 389 | for _ in range(self._size - 1): 390 | result = self.step(result, self._padding) 391 | else: 392 | while len(self._window) > 1: 393 | self._window.popleft() 394 | result = self._reducer.step(result, self._window_type(self._window)) 395 | return self._reducer.complete(result) 396 | 397 | 398 | def windowing(size, padding=UNSET, window_type=tuple): 399 | """Create a transducer which produces a moving window over items.""" 400 | 401 | if size < 1: 402 | raise ValueError("windowing() size {} is not at least 1".format(size)) 403 | 404 | def windowing_transducer(reducer): 405 | return Windowing(reducer, size, padding, window_type) 406 | 407 | return windowing_transducer 408 | 409 | # --------------------------------------------------------------------- 410 | 411 | 412 | class First(Transducer): 413 | 414 | def __init__(self, reducer, predicate): 415 | super().__init__(reducer) 416 | self._predicate = predicate 417 | 418 | def step(self, result, item): 419 | return Reduced(self._reducer.step(result, item)) if self._predicate(item) else result 420 | 421 | 422 | def first(predicate=None): 423 | """Create a transducer which obtains the first item, then terminates.""" 424 | 425 | predicate = true if predicate is None else predicate 426 | 427 | def first_transducer(reducer): 428 | return First(reducer, predicate) 429 | 430 | return first_transducer 431 | 432 | # --------------------------------------------------------------------- 433 | 434 | 435 | class Last(Transducer): 436 | 437 | def __init__(self, reducer, predicate): 438 | super().__init__(reducer) 439 | self._predicate = predicate 440 | self._last_seen = UNSET 441 | 442 | def step(self, result, item): 443 | if self._predicate(item): 444 | self._last_seen = item 445 | return result 446 | 447 | def complete(self, result): 448 | if self._last_seen is not UNSET: 449 | result = self._reducer.step(result, self._last_seen) 450 | return self._reducer.complete(result) 451 | 452 | 453 | def last(predicate=None): 454 | """Create a transducer which obtains the last item.""" 455 | 456 | predicate = true if predicate is None else predicate 457 | 458 | def last_transducer(reducer): 459 | return Last(reducer, predicate) 460 | 461 | return last_transducer 462 | 463 | # --------------------------------------------------------------------- 464 | 465 | 466 | class ElementAt(Transducer): 467 | 468 | def __init__(self, reducer, index): 469 | super().__init__(reducer) 470 | self._index = index 471 | self._counter = -1 472 | 473 | def step(self, result, item): 474 | self._counter += 1 475 | if self._counter == self._index: 476 | return Reduced(self._reducer.step(result, item)) 477 | return result 478 | 479 | def complete(self, result): 480 | if self._counter < self._index: 481 | raise IndexError("Too few elements in series of length {} " 482 | "to find element at index {}".format(self._counter, self._index)) 483 | return result 484 | 485 | 486 | def element_at(index): 487 | """Create a transducer which obtains the item at the specified index.""" 488 | 489 | if index < 0: 490 | raise IndexError("element_at used with illegal index {}".format(index)) 491 | 492 | def element_at_transducer(reducer): 493 | return ElementAt(reducer, index) 494 | 495 | return element_at_transducer 496 | 497 | # --------------------------------------------------------------------- 498 | 499 | 500 | class Repeating(Transducer): 501 | 502 | def __init__(self, reducer, num_times): 503 | super().__init__(reducer) 504 | self._num_times = num_times 505 | 506 | def step(self, result, item): 507 | for i in range(self._num_times): 508 | result = self._reducer.step(result, item) 509 | return result 510 | 511 | 512 | def repeating(num_times): 513 | 514 | if num_times < 0: 515 | raise ValueError("num_times value {} is not non-negative".format(num_times)) 516 | 517 | def repeating_transducer(reducer): 518 | return Repeating(reducer, num_times) 519 | 520 | return repeating_transducer 521 | 522 | # --------------------------------------------------------------------- 523 | 524 | 525 | class Reversing(Transducer): 526 | 527 | def __init__(self, reducer): 528 | super().__init__(reducer) 529 | self._items = deque() 530 | 531 | def step(self, result, item): 532 | self._items.appendleft(item) 533 | return result 534 | 535 | def complete(self, result): 536 | for item in self._items: 537 | result = self._reducer.step(result, item) 538 | 539 | self._items.clear() 540 | 541 | return self._reducer.complete(result) 542 | 543 | 544 | def reversing(): 545 | 546 | def reversing_transducer(reducer): 547 | return Reversing(reducer) 548 | 549 | return reversing_transducer 550 | 551 | # --------------------------------------------------------------------- 552 | 553 | 554 | class Ordering(Transducer): 555 | 556 | def __init__(self, reducer, key, reverse): 557 | super().__init__(reducer) 558 | self._key = key 559 | self._reverse = reverse 560 | self._items = [] 561 | 562 | def step(self, result, item): 563 | self._items.append(item) 564 | return result 565 | 566 | def complete(self, result): 567 | self._items.sort(key=self._key, reverse=self._reverse) 568 | 569 | for item in self._items: 570 | result = self._reducer.step(result, item) 571 | 572 | self._items.clear() 573 | 574 | return self._reducer.complete(result) 575 | 576 | 577 | def ordering(key=None, reverse=False): 578 | 579 | def ordering_transducer(reducer): 580 | return Ordering(reducer, key, reverse) 581 | 582 | return ordering_transducer 583 | 584 | # --------------------------------------------------------------------- 585 | 586 | 587 | class Counting(Transducer): 588 | 589 | def __init__(self, reducer, predicate): 590 | super().__init__(reducer) 591 | self._predicate = predicate 592 | self._count = 0 593 | 594 | def step(self, result, item): 595 | if self._predicate(item): 596 | self._count += 1 597 | return result 598 | 599 | def complete(self, result): 600 | result = self._reducer.step(result, self._count) 601 | return self._reducer.complete(result) 602 | 603 | 604 | def counting(predicate=None): 605 | 606 | predicate = true if predicate is None else predicate 607 | 608 | def counting_transducer(reducer): 609 | return Counting(reducer, predicate) 610 | 611 | return counting_transducer 612 | --------------------------------------------------------------------------------