├── CHANGELOG.md ├── requirements.txt ├── phi ├── tests │ ├── __init__.py │ ├── test.txt │ ├── some_module.py │ ├── test_state.py │ ├── test_dummy.py │ ├── test_nodsl.py │ ├── test_lambdas.py │ ├── test_dsl.py │ ├── test_builders.py │ └── test_examples.py ├── version.txt ├── api.py ├── __init__.py ├── utils.py ├── python_builder.py ├── README-template.md └── builder.py ├── guide ├── dsl │ ├── test.md │ └── README.md ├── patches │ ├── test.md │ └── README.md ├── scoping │ ├── test.md │ └── README.md ├── branching │ ├── test.md │ └── README.md ├── SUMMARY.md ├── README.md └── basics │ └── README.md ├── book.json ├── tasks ├── update_gh_pages ├── upload_live ├── upload_test ├── upload_new_version ├── test_pypitest ├── do_pypitest ├── test ├── gen_docs └── create_readme.py ├── MANIFEST.in ├── docker ├── python27.dockerfile └── python35.dockerfile ├── docker-compose.yml ├── LICENSE ├── setup.py ├── .gitignore ├── README.md └── docs └── phi └── tests ├── index.html └── some_module.m.html /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /phi/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /phi/version.txt: -------------------------------------------------------------------------------- 1 | 0.6.6 2 | -------------------------------------------------------------------------------- /guide/dsl/test.md: -------------------------------------------------------------------------------- 1 | # Some Test 2 | -------------------------------------------------------------------------------- /phi/tests/test.txt: -------------------------------------------------------------------------------- 1 | hello world -------------------------------------------------------------------------------- /guide/patches/test.md: -------------------------------------------------------------------------------- 1 | # Some Test 2 | -------------------------------------------------------------------------------- /guide/scoping/test.md: -------------------------------------------------------------------------------- 1 | # Some Test 2 | -------------------------------------------------------------------------------- /guide/branching/test.md: -------------------------------------------------------------------------------- 1 | # Some Test 2 | -------------------------------------------------------------------------------- /book.json: -------------------------------------------------------------------------------- 1 | { 2 | "root": "./guide" 3 | } 4 | -------------------------------------------------------------------------------- /guide/patches/README.md: -------------------------------------------------------------------------------- 1 | # Patches 2 | 3 | Coming Soon! 4 | -------------------------------------------------------------------------------- /guide/scoping/README.md: -------------------------------------------------------------------------------- 1 | # Scoping 2 | 3 | Coming Soon! 4 | -------------------------------------------------------------------------------- /tasks/update_gh_pages: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | git subtree push --prefix docs/phi origin gh-pages -------------------------------------------------------------------------------- /phi/tests/some_module.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | def some_fun(x): 4 | return "YES" 5 | 6 | def other_fun(x, y): 7 | return x / y -------------------------------------------------------------------------------- /tasks/upload_live: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # Register 4 | #python setup.py register -r pypi 5 | 6 | # Upload 7 | python setup.py sdist upload -r pypi 8 | -------------------------------------------------------------------------------- /tasks/upload_test: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # Register 4 | #python setup.py register -r pypitest 5 | 6 | # Upload 7 | python setup.py sdist upload -r pypitest 8 | -------------------------------------------------------------------------------- /tasks/upload_new_version: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | set -e 3 | 4 | tasks/gen_docs 5 | tasks/upload_test 6 | tasks/test_pypitest 7 | tasks/upload_live 8 | tasks/update_gh_pages -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENCE 2 | include requirements.txt 3 | include README.md 4 | include CHANGELOG.md 5 | 6 | include phi/version.txt 7 | include phi/README-template.md 8 | -------------------------------------------------------------------------------- /tasks/test_pypitest: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | set -e 3 | 4 | docker run -it -v ${PWD}:/code -w /code python:2.7 tasks/do_pypitest 5 | docker run -it -v ${PWD}:/code -w /code python:3.5 tasks/do_pypitest -------------------------------------------------------------------------------- /phi/tests/test_state.py: -------------------------------------------------------------------------------- 1 | from phi.api import * 2 | 3 | 4 | class TestState(object): 5 | 6 | def test_seq(self): 7 | 8 | f = Seq(lambda x: x + 1) 9 | 10 | assert f(1) == 2 11 | -------------------------------------------------------------------------------- /tasks/do_pypitest: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | set -e 3 | 4 | pip install -i https://testpypi.python.org/pypi phi 5 | echo 6 | echo "Testing Library:" 7 | echo 8 | python -c "import phi; print('Hello from python. Phi version:', phi.__version__)" -------------------------------------------------------------------------------- /guide/SUMMARY.md: -------------------------------------------------------------------------------- 1 | # Summary 2 | 3 | * [1. Basics](basics/README.md) 4 | * [Setup](basics/README.md#Setup) 5 | * [Building Networks](basics/README.md#Building_networks) 6 | * [2. Branching](branching/README.md) 7 | * [3. DSL](dsl/README.md) -------------------------------------------------------------------------------- /tasks/test: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | set -e 3 | 4 | echo "TESTING PYTHON27" 5 | find . -name "*.pyc" -delete 6 | docker-compose run --rm test27 7 | 8 | 9 | echo "TESTING PYTHON35" 10 | find . -name "*.pyc" -delete 11 | docker-compose run --rm test35 -------------------------------------------------------------------------------- /tasks/gen_docs: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | set -e 3 | 4 | find . -name "*.pyc" -delete 5 | echo "Generating docs" 6 | rm -fr docs 7 | PYTHONPATH=./phi pdoc --html-dir=docs --html --all-submodules --overwrite phi 8 | echo "Making README.md" 9 | python tasks/create_readme.py 10 | echo "Finished" 11 | -------------------------------------------------------------------------------- /tasks/create_readme.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | with open('phi/version.txt', 'r') as f: 5 | version = f.read() 6 | 7 | with open('phi/README-template.md', 'r') as f: 8 | readme = f.read().format(version) 9 | 10 | 11 | with open('README.md', 'w') as f: 12 | f.write(readme) 13 | 14 | -------------------------------------------------------------------------------- /docker/python27.dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:2.7 2 | 3 | RUN apt-get update 4 | RUN apt-get -y install git 5 | 6 | RUN pip install pdoc 7 | RUN pip install mako 8 | RUN pip install markdown 9 | RUN pip install decorator>=4.0.9 10 | RUN pip install pytest 11 | RUN pip install pytest-sugar 12 | RUN pip install ipdb 13 | -------------------------------------------------------------------------------- /docker/python35.dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.5 2 | 3 | RUN apt-get update 4 | RUN apt-get -y install git 5 | 6 | RUN pip install pdoc 7 | RUN pip install mako 8 | RUN pip install markdown 9 | RUN pip install decorator>=4.0.9 10 | RUN pip install pytest 11 | RUN pip install pytest-sugar 12 | RUN pip install ipdb 13 | -------------------------------------------------------------------------------- /phi/tests/test_dummy.py: -------------------------------------------------------------------------------- 1 | 2 | class A(object): 3 | """docstring for A.""" 4 | 5 | _S = None 6 | 7 | @property 8 | def S(self): 9 | return A._S 10 | 11 | class B(A): 12 | """docstring for B.""" 13 | 14 | class Compile(A): 15 | """docstring for Compile .""" 16 | 17 | 18 | class TestDummy(object): 19 | """docstring for TestDummy.""" 20 | 21 | def test_global(self): 22 | a = A() 23 | b = B() 24 | c = Compile() 25 | 26 | A._S = "HOLA" 27 | 28 | assert b.S == a.S + "" 29 | -------------------------------------------------------------------------------- /phi/tests/test_nodsl.py: -------------------------------------------------------------------------------- 1 | from phi.dsl import Expression 2 | 3 | P = Expression() 4 | 5 | class TestNoDSL(object): 6 | 7 | def test_seq(self): 8 | 9 | f = P.Seq( 10 | P + 2, 11 | P * 2 12 | ) 13 | 14 | assert 10 == f(3) 15 | 16 | def test_branch(self): 17 | 18 | f = P.List( 19 | P + 2, 20 | P * 2 21 | ) 22 | 23 | assert [5, 6] == f(3) 24 | 25 | f = P.List( 26 | P + 2, 27 | P.Seq( 28 | P + 1, 29 | P * 2 30 | ) 31 | ) 32 | 33 | assert [5, 8] == f(3) 34 | -------------------------------------------------------------------------------- /phi/api.py: -------------------------------------------------------------------------------- 1 | from .python_builder import P 2 | F = P.F 3 | 4 | #constants 5 | Val = P.Val 6 | 7 | #access 8 | Obj = P.Obj 9 | Rec = P.Rec 10 | 11 | #pipes 12 | Seq = P.Seq 13 | Pipe = P.Pipe 14 | 15 | #state 16 | Ref = P.Ref 17 | Read = P.Read 18 | Write = P.Write 19 | 20 | # flow control 21 | If = P.If 22 | With = P.With 23 | Context = P.Context 24 | 25 | # collections 26 | List = P.List 27 | Dict = P.Dict 28 | Set = P.Set 29 | Tuple = P.Tuple 30 | 31 | # special 32 | ReadList = P.ReadList 33 | 34 | # then 35 | Then0 = P.Then0 36 | Then = P.Then 37 | Then1 = P.Then1 38 | Then2 = P.Then2 39 | Then3 = P.Then3 40 | Then4 = P.Then4 41 | Then5 = P.Then5 42 | ThenAt = P.ThenAt 43 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | volumes: 4 | data: 5 | 6 | services: 7 | 8 | docs: 9 | # update gh-pages => git subtree push --prefix docs/phi origin gh-pages 10 | build: 11 | context: ./docker 12 | dockerfile: python27.dockerfile 13 | volumes: 14 | - ./:/phi 15 | working_dir: /phi 16 | command: ./tasks/gen_docs 17 | 18 | test27: 19 | build: 20 | context: ./docker 21 | dockerfile: python27.dockerfile 22 | volumes: 23 | - ./:/phi 24 | working_dir: /phi 25 | command: py.test -v -s -x --pdb 26 | 27 | test35: 28 | build: 29 | context: ./docker 30 | dockerfile: python35.dockerfile 31 | volumes: 32 | - ./:/phi 33 | working_dir: /phi 34 | command: py.test -v -s -x --pdb 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 cgarciae 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /phi/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | from __future__ import unicode_literals 5 | 6 | 7 | from . import utils 8 | from . import builder 9 | from . import dsl 10 | 11 | from .utils import identity 12 | 13 | from . import python_builder 14 | from .builder import Builder 15 | from .python_builder import PythonBuilder 16 | 17 | from .api import * 18 | 19 | ######################## 20 | # Documentation 21 | ######################## 22 | import os 23 | 24 | #set documentation 25 | 26 | def _to_pdoc_markdown(doc): 27 | indent = False 28 | lines = [] 29 | 30 | for line in doc.split('\n'): 31 | if "```" in line: 32 | indent = not indent 33 | line = line.replace("```python", '') 34 | line = line.replace("```", '') 35 | 36 | if indent: 37 | line = " " + line 38 | 39 | lines.append(line) 40 | 41 | return '\n'.join(lines) 42 | 43 | def _read(fname): 44 | return open(os.path.join(os.path.dirname(__file__), fname)).read() 45 | 46 | _raw_docs = _read("README-template.md") 47 | __version__ = _read("version.txt") 48 | __doc__ = _to_pdoc_markdown(_raw_docs.format(__version__)) 49 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from setuptools import setup 3 | 4 | 5 | # Utility function to read the README file. 6 | # Used for the long_description. It's nice, because now 1) we have a top level 7 | # README file and 2) it's easier to type in the README file than to put a raw 8 | # string in below ... 9 | def read(fname): 10 | with open(os.path.join(os.path.dirname(__file__), fname)) as f: 11 | return f.read() 12 | 13 | version = read('phi/version.txt').split("\n")[0] 14 | 15 | setup( 16 | name = "phi", 17 | version = version, 18 | author = "Cristian Garcia", 19 | author_email = "cgarcia.e88@gmail.com", 20 | description = ("Phi is a library for fluent functional programming in Python which includes a DSL + facilities to create libraries that integrate with it."), 21 | license = "MIT", 22 | keywords = ["functional programming", "DSL"], 23 | url = "https://github.com/cgarciae/phi", 24 | packages = [ 25 | 'phi', 26 | 'phi.tests' 27 | ], 28 | package_data={ 29 | '': ['LICENCE', 'requirements.txt', 'README.md', 'CHANGELOG.md'], 30 | 'phi': ['version.txt', 'README-template.md'] 31 | }, 32 | download_url = 'https://github.com/cgarciae/phi/tarball/{0}'.format(version), 33 | include_package_data = True, 34 | long_description = read('README.md'), 35 | install_requires = [], 36 | ) 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | tmp/ 2 | 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | *$py.class 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | env/ 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .coverage 43 | .coverage.* 44 | .cache 45 | nosetests.xml 46 | coverage.xml 47 | *,cover 48 | .hypothesis/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | 58 | # Flask stuff: 59 | instance/ 60 | .webassets-cache 61 | 62 | # Scrapy stuff: 63 | .scrapy 64 | 65 | # Sphinx documentation 66 | docs/_build/ 67 | 68 | # PyBuilder 69 | target/ 70 | 71 | # IPython Notebook 72 | .ipynb_checkpoints 73 | 74 | # pyenv 75 | .python-version 76 | 77 | # celery beat schedule file 78 | celerybeat-schedule 79 | 80 | # dotenv 81 | .env 82 | 83 | # virtualenv 84 | venv/ 85 | ENV/ 86 | 87 | # Spyder project settings 88 | .spyderproject 89 | 90 | # Rope project settings 91 | .ropeproject 92 | -------------------------------------------------------------------------------- /phi/tests/test_lambdas.py: -------------------------------------------------------------------------------- 1 | from phi.api import * 2 | 3 | class TestExpressions(object): 4 | """docstring for TestExpressions.""" 5 | 6 | def test_ops(self): 7 | 8 | assert 2 == P.Pipe( 9 | 1, 10 | P + 1 11 | ) 12 | 13 | ##################### 14 | ##################### 15 | 16 | f = P * [ P ] 17 | 18 | assert f(0) == [] 19 | assert f(1) == [1] 20 | assert f(3) == [3,3,3] 21 | 22 | def test_rshift(self): 23 | 24 | f = P + 1 >> P * 2 25 | 26 | assert 6 == f(2) 27 | 28 | 29 | f = lambda x: x * 3 30 | g = lambda x: x + 2 31 | 32 | h = f >> P.Seq(g) 33 | 34 | assert 11 == h(3) 35 | 36 | 37 | h = P.Seq(f) >> g 38 | assert 11 == h(3) 39 | 40 | 41 | y = 1 >> P + 1 >> P * 2 42 | assert 4 == y 43 | 44 | 45 | y = P * 2 << P + 1 << 1 46 | assert 4 == y 47 | 48 | def test_lshift(self): 49 | 50 | f = P * 2 << P + 1 51 | 52 | assert 6 == f(2) 53 | 54 | 55 | f = lambda x: x * 3 56 | g = lambda x: x + 2 57 | 58 | h = g << Seq(f) 59 | 60 | assert 11 == h(3) 61 | 62 | 63 | h = Seq(g) << f 64 | 65 | assert 11 == h(3) 66 | 67 | def test_reverse(self): 68 | 69 | f = 12 / P 70 | 71 | assert f(3) == 4 72 | 73 | def test_lambda_opt_lambda(self): 74 | 75 | assert 3 == Pipe( 76 | 0, 77 | List( 78 | P + 1 79 | , 80 | P + 2 81 | ), 82 | P[0] + P[1] 83 | ) 84 | 85 | assert 3 == P.Pipe( 86 | Dict( 87 | a = 1, 88 | b = 2 89 | ), 90 | Rec.a + Rec.b 91 | ) 92 | 93 | assert 5 == P.Pipe( 94 | Dict( 95 | a = 10, 96 | b = 2 97 | ), 98 | Rec.a / Rec.b 99 | ) 100 | 101 | 102 | assert 6 == 1 >> (P + 1) * (P + 2) 103 | 104 | assert 6 == 10 >> (P * 3) / (P - 5) 105 | -------------------------------------------------------------------------------- /guide/branching/README.md: -------------------------------------------------------------------------------- 1 | # Branching 2 | 3 | Branching is common in many neural networks that need to resolve complex tasks because each branch to specialize its knowledge while lowering number of weight compared to a network with wider layers, thus giving better performance. TensorBuilder enables you to easily create nested branches. Branching results in a `BuilderTree`, which has methods for traversing all the `Builder` leaf nodes and reducing the whole tree to a single `Builder`. 4 | 5 | To create a branch you just have to use the `Builder.branch` method 6 | 7 | import tensorflow as tf 8 | from phi import tb 9 | 10 | x = tf.placeholder(tf.float32, shape=[None, 5]) 11 | keep_prob = tf.placeholder(tf.float32) 12 | 13 | h = ( 14 | tb.build(x) 15 | .fully_connected(10) 16 | .branch(lambda root: 17 | [ 18 | root 19 | .fully_connected(3, activation_fn=tf.nn.relu) 20 | , 21 | root 22 | .fully_connected(9, activation_fn=tf.nn.tanh) 23 | .branch(lambda root2: 24 | [ 25 | root2 26 | .fully_connected(6, activation_fn=tf.nn.sigmoid) 27 | , 28 | root2 29 | .map(tf.nn.dropout, keep_prob) 30 | .fully_connected(8, tf.nn.softmax) 31 | ]) 32 | ]) 33 | .fully_connected(6, activation_fn=tf.nn.sigmoid) 34 | .tensor() 35 | ) 36 | 37 | print(h) 38 | 39 | Thanks to TensorBuilder's immutable API, each branch is independent. The previous can also be simplified with the full `patch` 40 | 41 | import tensorflow as tf 42 | from phi import tb 43 | import phi.patch 44 | 45 | x = tf.placeholder(tf.float32, shape=[None, 5]) 46 | keep_prob = tf.placeholder(tf.float32) 47 | 48 | h = ( 49 | tb.build(x) 50 | .fully_connected(10) 51 | .branch(lambda root: 52 | [ 53 | root 54 | .relu_layer(3) 55 | , 56 | root 57 | .tanh_layer(9) 58 | .branch(lambda root2: 59 | [ 60 | root2 61 | .sigmoid_layer(6) 62 | , 63 | root2 64 | .dropout(keep_prob) 65 | .softmax_layer(8) 66 | ]) 67 | ]) 68 | .sigmoid_layer(6) 69 | .tensor() 70 | ) 71 | 72 | print(h) 73 | -------------------------------------------------------------------------------- /guide/dsl/README.md: -------------------------------------------------------------------------------- 1 | # DSL 2 | 3 | TensorBuilder's DSL enables you to express the computation do desire to do into a single flexible structure. The DSL preserves all features of given to you by the `Builder` class: 4 | 5 | * Composing operations 6 | * Branching 7 | * Scoping 8 | 9 | The `Applicative` was built to create elements that are accepted/play well will this language. It also two very import methods 10 | 11 | * `compile`: generates a function out a given valid **ast**/structure (compiles it) 12 | * `pipe`: given `Builder` or `Tensor` and an **ast**, compile it to a function and apply it to the Tensor/Builder. 13 | 14 | ## Rules 15 | 16 | * All final elements in the "AST" must be functions, non final elements are compiled to a function. 17 | * A Tuple `()` denotes a sequential operation. Results in the composition of all elements within it. 18 | * A List `[]` denotes a branching operation. Results in the creation of a function that applies the `.branch` method to its argument, and each element in the list results in a branch. It compiles to a function of type `Builder -> BuilderTree`. 19 | * A Dict `{}` denotes a scoping operation. It only accepts a single key-value pair, its key must me a [Disposable](https://www.python.org/dev/peps/pep-0343/) and its value can be any element of the language. It results in the creation of a function that takes a `Builder` as its argument, applies the `with` statemente to the `key` and applies the function of the `value` to its argument inside the `with` block. 20 | 21 | ## Example 22 | 23 | Its easier to see the actual DSL with an example, especially because you can see a direct mapping of the concepts brought by the `Builder` class into the DSL: 24 | 25 | import tensorflow as tf 26 | from phi import tb 27 | 28 | x = placeholder(tf.float32, shape=[None, 10]) 29 | y = placeholder(tf.float32, shape=[None, 5]) 30 | 31 | [h, trainer] = tb.pipe( 32 | x, 33 | [ 34 | { tf.device("/gpu:0"): 35 | tb.relu_layer(20) 36 | } 37 | , 38 | { tf.device("/gpu:1"): 39 | tb.sigmoid_layer(20) 40 | } 41 | , 42 | { tf.device("/cpu:0"): 43 | tb.tanh_layer(20) 44 | } 45 | ], 46 | tb.relu_layer(10) 47 | .linear_layer(5), 48 | [ 49 | tb.softmax() # h 50 | , 51 | tb.softmax_cross_entropy_with_logits(y) 52 | .reduce_mean() 53 | .map(tf.trainer.AdamOptimizer(0.01).minimize) # trainer 54 | ], 55 | tb.tensors() 56 | ) 57 | 58 | Lets go step by step to what is happening here: 59 | 60 | 1. The Tensor `x` pluged inside a `Builder` and *piped* through the computational structured defined. All the arguments of `pipe` after `x` are grouped as if they were in a tuple `()` and the whole expression is compiled to a single function with is then applied to the `Buider` containing `x`. 61 | 1. **final** elements you see here like `tb.softmax()` are `Applicative`s which as you've been told are functions. As you see, *almost* all methods from the `Builder` class are also methods from the `Applicative` class, the diference is that the methods of the `Builder` class actually perform the computation they intend (construct a new Tensor), but the methods from the `Applicative` class rather *compose/define* the computation to be done later. 62 | 1. There is an implicit Tuple `()` element that is performing a sequential composition of all the other elements. As a result, the visual/spatial ordering of the code corresponds to the intended behavior. 63 | 1. Lists very naturally express branches. Notice how indentation and an intentional positioning of the `,` comma help to diferentiate each branch. 64 | 1. Expresions like `tb.relu_layer(10)` are polymorphic and work for `Builder`s or `BuilderTree`s regardless. 65 | 1. Scoping is very clean with the `{}` notation. In constrast to using `then_with` from the `Builder` class, here you can actually use the original functions from `tensorflow` unchanged in the `key` of the dict. -------------------------------------------------------------------------------- /phi/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | """ 3 | 4 | from __future__ import absolute_import 5 | from __future__ import division 6 | from __future__ import print_function 7 | from __future__ import unicode_literals 8 | 9 | from collections import namedtuple 10 | import inspect 11 | 12 | def identity(x): 13 | return x 14 | 15 | def state_identity(x, state): 16 | return x, state 17 | 18 | def compose2(f, g): 19 | return lambda x: f(g(x)) 20 | 21 | def forward_compose2(f, g): 22 | return lambda x: g(f(x)) 23 | 24 | def merge(dict_a, dict_b): 25 | return dict(dict_a, **dict_b) 26 | 27 | def lift(f): 28 | return lambda x, state: (f(x), state) 29 | 30 | 31 | class _NoValue(object): 32 | def __repr__(self): 33 | return "NoValue" 34 | 35 | NO_VALUE = _NoValue() 36 | 37 | DefaultArgSpec = namedtuple('DefaultArgSpec', 'has_default default_value') 38 | 39 | def _get_default_arg(args, defaults, arg_index): 40 | """ Method that determines if an argument has default value or not, 41 | and if yes what is the default value for the argument 42 | 43 | :param args: array of arguments, eg: ['first_arg', 'second_arg', 'third_arg'] 44 | :param defaults: array of default values, eg: (42, 'something') 45 | :param arg_index: index of the argument in the argument array for which, 46 | this function checks if a default value exists or not. And if default value 47 | exists it would return the default value. Example argument: 1 48 | :return: Tuple of whether there is a default or not, and if yes the default 49 | value, eg: for index 2 i.e. for "second_arg" this function returns (True, 42) 50 | """ 51 | if not defaults: 52 | return DefaultArgSpec(False, None) 53 | 54 | args_with_no_defaults = len(args) - len(defaults) 55 | 56 | if arg_index < args_with_no_defaults: 57 | return DefaultArgSpec(False, None) 58 | else: 59 | value = defaults[arg_index - args_with_no_defaults] 60 | if (type(value) is str): 61 | value = '"%s"' % value 62 | return DefaultArgSpec(True, value) 63 | 64 | def get_method_sig(method): 65 | """ Given a function, it returns a string that pretty much looks how the 66 | function signature_ would be written in python. 67 | 68 | :param method: a python method 69 | :return: A string similar describing the pythong method signature_. 70 | eg: "my_method(first_argArg, second_arg=42, third_arg='something')" 71 | """ 72 | 73 | # The return value of ArgSpec is a bit weird, as the list of arguments and 74 | # list of defaults are returned in separate array. 75 | # eg: ArgSpec(args=['first_arg', 'second_arg', 'third_arg'], 76 | # varargs=None, keywords=None, defaults=(42, 'something')) 77 | argspec = inspect.getargspec(method) 78 | arg_index=0 79 | args = [] 80 | 81 | # Use the args and defaults array returned by argspec and find out 82 | # which arguments has default 83 | for arg in argspec.args: 84 | default_arg = _get_default_arg(argspec.args, argspec.defaults, arg_index) 85 | if default_arg.has_default: 86 | args.append("%s=%s" % (arg, default_arg.default_value)) 87 | else: 88 | args.append(arg) 89 | arg_index += 1 90 | return "%s(%s)" % (method.__name__, ", ".join(args)) 91 | 92 | def get_instance_methods(instance): 93 | for method_name in dir(instance): 94 | method = getattr(instance, method_name) 95 | if hasattr(method, '__call__'): 96 | yield method_name, method 97 | 98 | 99 | def _flatten_list(container): 100 | for i in container: 101 | if isinstance(i, list): 102 | for j in flatten_list(i): 103 | yield j 104 | else: 105 | yield i 106 | 107 | def flatten_list(container): 108 | return list(_flatten_list(container)) 109 | 110 | 111 | def _flatten(container): 112 | for i in container: 113 | if hasattr(i, '__iter__'): 114 | for j in _flatten(i): 115 | yield j 116 | else: 117 | yield i 118 | 119 | def flatten(container): 120 | return list(_flatten(container)) 121 | -------------------------------------------------------------------------------- /phi/python_builder.py: -------------------------------------------------------------------------------- 1 | """ 2 | `PythonBuilder` helps you integrate Python's built-in functions and keywords into the DSL and it also includes a bunch of useful helpers for common stuff. `phi`'s global `P` object is an instance of this class. 3 | """ 4 | 5 | from __future__ import absolute_import 6 | from __future__ import division 7 | from __future__ import print_function 8 | from __future__ import unicode_literals 9 | 10 | 11 | from .builder import Builder 12 | from . import utils 13 | import inspect 14 | 15 | class PythonBuilder(Builder): 16 | """ 17 | This class has two types of methods: 18 | 19 | 1. Methods that start with a lowercase letter are core python functions automatically registered as methods (e.g. `phi.python_builder.PythonBuilder.map` or `phi.python_builder.PythonBuilder.sum`). 20 | 2. Methods that start with a capytal letter like `phi.python_builder.PythonBuilder.And`, `phi.python_builder.PythonBuilder.Not`, `phi.python_builder.PythonBuilder.Contains`, this is done because some mimimic keywords (`and`, `or`, `not`, etc) and its ilegal to give them these lowercase names, however, methods like `phi.python_builder.PythonBuilder.Contains` that could use lowercase are left capitalized to maintain uniformity. 21 | """ 22 | 23 | P = PythonBuilder() 24 | 25 | # built in functions 26 | _function_2_names = ["map", "filter", "reduce"] 27 | _functions_2 = [ (_name, f) for _name, f in __builtins__.items() if _name in _function_2_names ] 28 | 29 | for _name, f in __builtins__.items(): 30 | try: 31 | if hasattr(f, "__name__") and _name[0] is not "_" and not _name[0].isupper() and _name not in _function_2_names: 32 | PythonBuilder.Register(f, "", alias=_name) 33 | except Exception as e: 34 | print(e) 35 | 36 | for _name, f in _functions_2: 37 | PythonBuilder.Register2(f, "") 38 | 39 | #custom methods 40 | @PythonBuilder.Register("phi.python_builder.", explain=False) 41 | def Not(a): 42 | """ 43 | **Not** 44 | 45 | Not() <=> lambda a: not a 46 | 47 | Returns a function that negates the input argument. 48 | 49 | ** Examples ** 50 | 51 | from phi import P 52 | 53 | assert True == P.Pipe( 54 | 1, 55 | P + 1, # 1 + 1 == 2 56 | P > 5, # 2 > 5 == False 57 | P.Not() # not False == True 58 | ) 59 | 60 | or shorter 61 | 62 | from phi import P 63 | 64 | assert True == P.Pipe( 65 | 1, 66 | (P + 1 > 5).Not() # not 1 + 1 > 5 == not 2 > 5 == not False == True 67 | ) 68 | 69 | or just 70 | 71 | from phi import P 72 | 73 | f = (P + 1 > 5).Not() 74 | 75 | assert f(1) == True 76 | """ 77 | return not a 78 | 79 | @PythonBuilder.Register("phi.python_builder.", explain=False) 80 | def Contains(a, b): 81 | """ 82 | **Contains** 83 | 84 | Contains(b) <=> lambda a: b in a 85 | 86 | Returns a partial function which when executed determines whether the argument partially applied is contained in the value being passed down. 87 | 88 | ** Examples ** 89 | 90 | from phi import P 91 | 92 | assert False == P.Pipe( 93 | [1,2,3,4], P 94 | .filter(P % 2 != 0) #[1, 3], keeps odds 95 | .Contains(4) #4 in [1, 3] == False 96 | ) 97 | """ 98 | return b in a 99 | 100 | @PythonBuilder.Register("phi.python_builder.", explain=False) 101 | def In(a, b): 102 | """ 103 | **In** 104 | 105 | In(b) <=> lambda a: a in b 106 | 107 | Returns a partial function which when executed determines whether the argument partially applied contains the value being passed down. 108 | 109 | ** Examples ** 110 | 111 | from phi import P 112 | 113 | assert False == P.Pipe( 114 | 3, 115 | P * 2, #3 * 2 == 6 116 | P.In([1,2,3,4]) #6 in [1,2,3,4] == False 117 | ) 118 | """ 119 | return a in b 120 | 121 | @PythonBuilder.Register("phi.python_builder.", explain=False) 122 | def First(a): 123 | """ 124 | **First** 125 | 126 | First() <=> lambda a: a[0] 127 | 128 | Returns a function which when executed returns the first element of the iterable being passed. 129 | 130 | ** Examples ** 131 | 132 | from phi import P 133 | 134 | assert 3 == P.Pipe( 135 | range(1, 10), P #[1, 2, ..., 8, 9] 136 | .filter(P % 3 == 0) #[3, 6, 9] 137 | .First() # [3, 6, 9][0] == 3 138 | ) 139 | """ 140 | return next(iter(a)) 141 | 142 | @PythonBuilder.Register("phi.python_builder.", explain=False) 143 | def Last(a): 144 | """ 145 | **Last** 146 | 147 | Last() <=> lambda a: a[-1] 148 | 149 | Returns a function which when executed returns the last element of the iterable being passed. 150 | 151 | ** Examples ** 152 | 153 | from phi import P 154 | 155 | assert 3 == P.Pipe( 156 | range(1, 10), P #[1, 2, ..., 8, 9] 157 | .filter(P % 3 == 0) #[3, 6, 9] 158 | .Last() # [3, 6, 9][-1] == 9 159 | ) 160 | """ 161 | return list(a)[-1] 162 | 163 | @PythonBuilder.Register("phi.python_builder.", explain=False) 164 | def Flatten(a): 165 | return utils.flatten(a) 166 | 167 | 168 | 169 | __all__ = ["PythonBuilder"] 170 | -------------------------------------------------------------------------------- /guide/README.md: -------------------------------------------------------------------------------- 1 | # Phi 2 | Python is a very nice language that favors readability but its not very strong at functional programming and this often leads to repetitive code. Phi intends to remove as much of the pain as possible from your functional programming experience in Python by providing the following modules 3 | 4 | * [dsl](https://cgarciae.github.io/phi/dsl.m.html): a small DSL that helps you compose computations in various ways & more. 5 | * [lambdas](https://cgarciae.github.io/phi/lambdas.m.html): easy way to create quick lambdas with a mathematical flavor. 6 | * [builder](https://cgarciae.github.io/phi/builder.m.html): an extensible class that enables you to integrate other libraries into the DSL through a [fluent](https://en.wikipedia.org/wiki/Fluent_interface) API. 7 | * [patch](https://cgarciae.github.io/phi/patch.m.html): this module contains some helpers that enable you to integrate a complete existing module or class into the DSL be registering its methods/functions into a [Builder](https://cgarciae.github.io/phi/builder.m.html#phi.builder.Builder). 8 | 9 | ## Documentation 10 | Check out the [complete documentation](https://cgarciae.github.io/phi/). 11 | 12 | ## Getting Started 13 | The global `phi.P` object exposes most of the API and preferably should be imported directly. The most simple thing the DSL does is function composition: 14 | 15 | 16 | from phi import P 17 | 18 | def add1(x): return x + 1 19 | def mul3(x): return x * 3 20 | 21 | x = P.Pipe( 22 | 1, 23 | add1, 24 | mul3 25 | ) 26 | 27 | assert x == 6 28 | 29 | 30 | The previous using [lambdas](https://cgarciae.github.io/phi/lambdas.m.html) 31 | 32 | 33 | from phi import P 34 | 35 | x = P.Pipe( 36 | 1, 37 | P + 1, 38 | P * 3 39 | ) 40 | 41 | assert x == 6 42 | 43 | 44 | Create a branched computation instead 45 | 46 | 47 | from phi import P 48 | 49 | [x, y] = P.Pipe( 50 | 1, 51 | [ 52 | P + 1 #1 + 1 == 2 53 | , 54 | P * 3 #1 * 3 == 3 55 | ] 56 | ) 57 | 58 | assert x == 2 59 | assert y == 3 60 | 61 | 62 | Compose it with a multiplication by 2 63 | 64 | 65 | from phi import P 66 | 67 | [x, y] = P.Pipe( 68 | 1, 69 | P * 2, #1 * 2 == 2 70 | [ 71 | P + 1 #2 + 1 == 3 72 | , 73 | P * 3 #2 * 3 == 6 74 | ] 75 | ) 76 | 77 | assert x == 3 78 | assert y == 6 79 | 80 | 81 | Give names to the branches 82 | 83 | 84 | from phi import P 85 | 86 | result = P.Pipe( 87 | 1, 88 | P * 2, #1 * 2 == 2 89 | dict( 90 | x = P + 1 #2 + 1 == 3 91 | , 92 | y = P * 3 #2 * 3 == 6 93 | ) 94 | ) 95 | 96 | assert result.x == 3 97 | assert result.y == 6 98 | 99 | 100 | Divide the `x` by the `y`. 101 | 102 | 103 | from phi import P, Rec 104 | 105 | result = P.Pipe( 106 | 1, 107 | P * 2, #1 * 2 == 2 108 | dict( 109 | x = P + 1 #2 + 1 == 3 110 | , 111 | y = P * 3 #2 * 3 == 6 112 | ), 113 | Rec.x / Rec.y #3 / 6 == 0.5 114 | ) 115 | 116 | assert result == 0.5 117 | 118 | 119 | Save the value from the `P * 2` computation as `s` and retrieve it at the end in a branch 120 | 121 | 122 | from phi import P, Rec 123 | 124 | [result, s] = P.Pipe( 125 | 1, 126 | P * 2, {'s'} #1 * 2 == 2 127 | dict( 128 | x = P + 1 #2 + 1 == 3 129 | , 130 | y = P * 3 #2 * 3 == 6 131 | ), 132 | [ 133 | Rec.x / Rec.y #3 / 6 == 0.5 134 | , 135 | 's' 136 | ] 137 | ) 138 | 139 | assert result == 0.5 140 | assert s == 2 141 | 142 | 143 | Add an input `Val` of 9 on a branch and add to it 1 just for the sake of it 144 | 145 | 146 | from phi import P, Rec, Val 147 | 148 | [result, s, val] = P.Pipe( 149 | 1, 150 | P * 2, {'s'} #2 * 1 == 2 151 | dict( 152 | x = P + 1 #2 + 1 == 3 153 | , 154 | y = P * 3 #2 * 3 == 6 155 | ), 156 | [ 157 | Rec.x / Rec.y #3 / 6 == 0.5 158 | , 159 | 's' #2 160 | , 161 | Val(9) + 1 #10 162 | ] 163 | ) 164 | 165 | assert result == 0.5 166 | assert s == 2 167 | assert val == 10 168 | 169 | 170 | ## Installation 171 | 172 | pip install phi 173 | 174 | 175 | #### Bleeding Edge 176 | 177 | pip install git+https://github.com/cgarciae/phi.git@develop 178 | 179 | ## Status 180 | * Version: **0.2.1**. 181 | * Current effort: Documentation (> 60%). Please create an issue if documentation is unclear, its of great priority for this library. 182 | * Milestone: reach 1.0.0 after docs completed + feedback from the community. 183 | 184 | ## Nice Examples 185 | 186 | 187 | from phi import P, Obj 188 | 189 | avg_word_length = P.Pipe( 190 | "1 22 33", 191 | Obj.split(" "), # ['1', '22', '333'] 192 | P.map(len), # [1, 2, 3] 193 | P.sum() / P.len() # sum([1,2,3]) / len([1,2,3]) == 6 / 3 == 2 194 | ) 195 | 196 | assert 2 == avg_word_length 197 | 198 | -------------------------------------------------------------------------------- /phi/tests/test_dsl.py: -------------------------------------------------------------------------------- 1 | from phi.api import * 2 | from phi import dsl 3 | import pytest 4 | 5 | class TestDSL(object): 6 | """docstring for TestDSL.""" 7 | 8 | def test_compile(self): 9 | f = Seq(P + 1, P * 2) 10 | assert f(2) == 6 11 | 12 | def test_read(self): 13 | refs = dict( 14 | x = 10 15 | ) 16 | 17 | f = Read('x') 18 | 19 | y, new_refs = f(None, True, **refs) 20 | 21 | assert refs == new_refs #read doesnt modify 22 | assert y == 10 23 | 24 | def test_write(self): 25 | r = dsl.Ref('r') 26 | f = Seq( 27 | Write(a = P + 1), 28 | Write(b = P * 2), 29 | Write(c = P * 100), 30 | List(Read.c, Read.a, Read.b) 31 | ) 32 | 33 | assert [600, 3, 6] == f(2) 34 | 35 | r = Ref('r') 36 | f = Seq( 37 | Write(a = P + 1), 38 | Write(b = P * 2), 39 | Write(c = P * 100), r.write, 40 | List(Read.c, Read.a, Read.b) 41 | ) 42 | 43 | assert [600, 3, 6] == f(2) 44 | assert r() == 600 45 | 46 | def test_write_tree(self): 47 | 48 | f = Seq( 49 | P + 1, 50 | P * 2, 51 | List( 52 | Write(c = P * 100) 53 | , 54 | P - 3 55 | , 56 | Read.c 57 | ) 58 | ) 59 | 60 | assert [600, 3, 600] == f(2) 61 | 62 | def test_write_tree(self): 63 | 64 | f = Seq( 65 | P + 1, 66 | P * 2, 67 | List( 68 | P * 100 69 | , 70 | Write(c = P) 71 | , 72 | P - 3 73 | , 74 | Read('c') 75 | ) 76 | ) 77 | 78 | assert [600, 6, 3, 6] == f(2) 79 | 80 | def test_input(self): 81 | f = Seq( 82 | Write(a = P), 83 | P + 1, 84 | List( 85 | Seq( 86 | 10, 87 | P * 2 88 | ) 89 | , 90 | Read('a') 91 | , 92 | P 93 | ) 94 | ) 95 | 96 | assert [20, 2, 3] == f(2) 97 | 98 | def test_identities(self): 99 | 100 | f = List( 101 | Seq(), 102 | List() 103 | ) 104 | 105 | assert [4, []] == f(4) 106 | 107 | def test_single_functions(self): 108 | 109 | f = List( 110 | P * 2, 111 | List(P + 1) 112 | ) 113 | 114 | assert [2, [2]] == f(1) 115 | 116 | def test_class(self): 117 | 118 | f = Seq( 119 | str, 120 | P + '0', 121 | int 122 | ) 123 | assert 20 == f(2) 124 | 125 | 126 | ast = dsl._parse(str) 127 | assert type(ast) is dsl.Expression 128 | 129 | def test_list(self): 130 | f = Seq( 131 | List( 132 | P + 1 133 | , 134 | P * 2 135 | ), 136 | List( 137 | Seq( 138 | lambda l: map(str, l), 139 | list 140 | ) 141 | , 142 | P 143 | ) 144 | ) 145 | 146 | assert [['4', '6'], [4, 6]] == f(3) 147 | 148 | def test_dict(self): 149 | f = Seq( 150 | Dict( 151 | original = P, 152 | upper = Obj.upper(), 153 | len = len 154 | ), 155 | List( 156 | P 157 | , 158 | Seq( 159 | Rec.len, 160 | P * 2 161 | ) 162 | ) 163 | ) 164 | 165 | [obj, double_len] = f("hello") 166 | 167 | assert obj.original == "hello" 168 | assert obj.upper == "HELLO" 169 | assert obj.len == 5 170 | assert double_len == 10 171 | 172 | def test_fn(self): 173 | 174 | assert "hola" == P.Pipe( 175 | "HOLA", 176 | Obj.lower() 177 | ) 178 | 179 | def test_record_object(self): 180 | 181 | x = P.Pipe( 182 | [1,2,3], 183 | Dict( 184 | sum = sum 185 | , 186 | len = len 187 | ) 188 | ) 189 | 190 | assert x.sum == 6 191 | assert x.len == 3 192 | 193 | assert x['sum'] == 6 194 | assert x['len'] == 3 195 | 196 | def test_compile_refs(self): 197 | 198 | x = P.Pipe( 199 | [1,2,3], 200 | Dict( 201 | sum = sum 202 | , 203 | len = len 204 | , 205 | x = Read.x 206 | , 207 | z = Read('y') + 2 208 | ), 209 | refs = dict( 210 | x = 10, 211 | y = 5 212 | ) 213 | ) 214 | 215 | assert x.sum == 6 216 | assert x.len == 3 217 | assert x.x == 10 218 | assert x.z == 7 219 | 220 | assert x['sum'] == 6 221 | assert x['len'] == 3 222 | assert x['x'] == 10 223 | assert x['z'] == 7 224 | 225 | ############################# 226 | 227 | 228 | f = P.Seq( 229 | If( P > 2, 230 | Write(s = P) 231 | ), 232 | Read('s') 233 | ) 234 | 235 | assert f(3) == 3 236 | 237 | with pytest.raises(Exception): 238 | f(1) 239 | 240 | 241 | def test_nested_compiles(self): 242 | 243 | assert 2 == P.Pipe( 244 | 1, Write(s = P), 245 | Seq( 246 | Write(s = P + 1) 247 | ), 248 | Write(s = P) 249 | ) 250 | 251 | def test_if(self): 252 | 253 | f = P.Seq( 254 | If( P > 0, 255 | P 256 | ).Else( 257 | 0 258 | ) 259 | ) 260 | 261 | assert f(5) == 5 262 | assert f(-3) == 0 263 | 264 | def test_right_hand(self): 265 | 266 | f = Seq( 267 | P + 1, 268 | [ P, 2, 3 ] 269 | ) 270 | assert f(0) == [ 1, 2, 3 ] 271 | 272 | f = Seq( 273 | P + 1, 274 | ( P, 2, 3 ) 275 | ) 276 | assert f(0) == ( 1, 2, 3 ) 277 | 278 | f = Seq( 279 | P + 1, 280 | { P, 2, 3 } 281 | ) 282 | assert f(0) == { 1, 2, 3 } 283 | 284 | 285 | f = Seq( 286 | P + 1, 287 | {"a": P, "b": 2, "c": 3 } 288 | ) 289 | assert f(0) == {"a": 1, "b": 2, "c": 3 } 290 | 291 | 292 | def test_readlist(self): 293 | 294 | assert [2, 4, 22] == Pipe( 295 | 1, 296 | Write(a = P + 1), #a = 1 + 1 == 2 297 | Write(b = P * 2), #b = 2 * 2 == 4 298 | P * 5, # 4 * 5 == 20 299 | P + 2, # 20 + 2 == 22 300 | ReadList('a', 'b', P) # [a, b, 22] == [2, 4, 22] 301 | ) 302 | -------------------------------------------------------------------------------- /guide/basics/README.md: -------------------------------------------------------------------------------- 1 | # Basics 2 | Here we will cover the basics of Tensor Builder, for this we will solve one of the simplest classical examples in the history of neural network: the XOR. 3 | 4 | We will assume that you have already installed TensorBuilder, if not click [here](https://cgarciae.gitbooks.io/phi/content/). Remember that you must have a working installation of TensorFlow. 5 | 6 | ## Setup 7 | First we will setup our imports, you'll need to have `numpy` installed. 8 | 9 | ```python 10 | import numpy as np 11 | 12 | import tensorflow as tf 13 | from phi import tb 14 | ``` 15 | 16 | As you see `tb` is **not** an alias for the `phi` module, its actually an object that we import from this library. There are several reason behind exposing the API as an object, one is that implementing it this way reduced a lot of code internally, but it also plays better with the DSL as you might see later. 17 | 18 | > **Note:** `tb` is of type `Applicative` and all of its methods are immutable, so down worry about "breaking" it. 19 | 20 | Next we are going to create our data and placeholders 21 | 22 | ```python 23 | #TRUTH TABLE (DATA) 24 | X = [[0.0,0.0]]; Y = [[0.0]] 25 | X.append([1.0,0.0]); Y.append([1.0]) 26 | X.append([0.0,1.0]); Y.append([1.0]) 27 | X.append([1.0,1.0]); Y.append([0.0]) 28 | 29 | X = np.array(X) 30 | Y = np.array(Y) 31 | 32 | x = tf.placeholder(tf.float32, shape=[None, 2]) 33 | y = tf.placeholder(tf.float32, shape=[None, 1]) 34 | ``` 35 | 36 | ## Building Networks 37 | Now we need to construct smallest the neural network that can solve the XOR, its architecture is going to be `[2 input, 2 sigmoid, 1 sigmoid]`. To do that we will first calculate the `logit` of the last layer, and then using it we will calculate 2 things: 38 | 39 | 1. The `activation` function (sometimes denoted `h`) by using the `sigmoid` function 40 | 2. The networks `trainer` by creating a loss function and feeding it to a training algorithm. 41 | 42 | Here is the code 43 | 44 | ```python 45 | logit = ( 46 | tb 47 | .build(x) 48 | .sigmoid_layer(2) 49 | .linear_layer(1) 50 | ) 51 | 52 | activation = ( 53 | logit 54 | .sigmoid() 55 | .tensor() 56 | ) 57 | 58 | trainer = ( 59 | logit 60 | .sigmoid_cross_entropy_with_logits(y) # loss 61 | .map(tf.train.AdamOptimizer(0.01).minimize) 62 | .tensor() 63 | ) 64 | ``` 65 | 66 | As you see `TensorBuilder`s API is fluent, meaning that you can keep chaining methods to *build* the computation. 67 | 68 | ### The Builder class 69 | The first thing we should talk about when reviewing this code is the `Builder` class. When we executed 70 | 71 | ```python 72 | tb 73 | .build(x) 74 | ``` 75 | 76 | we created a `Builder` that holds our input Tensor `x`. Having our `Builder` we proceeded to use the methods 77 | 78 | ```python 79 | .sigmoid_layer(2) 80 | .linear_layer(1) 81 | ``` 82 | 83 | If the acronym "What You Read is Mostly What You Get (WYRMWYG)" were a thing, this code would be it. Its telling you that the input is connected to a layer of *2 sigmoid* units, and then this is connected to a layer of *1 linear* unit. You might be wondering where do these methods come from? Or what kinds of methods are there? 84 | 85 | #### Method Families 86 | 87 | `TensorBuilder` decided to become a library that doesn't implement the core methods that actually deal with Tensors. Instead it has some class methods to **register** instance methods and during import we actually include a bunch of functions from other libraries (yeah we are basically just stealing other libraries for the greater good). Currently most of these methods come from the `tensorflow` library, but there are also some from `tflearn`. The the current practice is the following 88 | 89 | 1. The function `tf.contrib.layers.fully_connected` is a very special function that is registered as a method of this class. Its importance is due to the fact that the most fundamental operations in the creation of neural networks involve creating/connecting layers. 90 | 2. If `f` is a funciton in `tf` or `tf.nn`, it will *most likely* be registered as method of the `Builder` class. The process that registers these functions *lifts* them from being functions that accept a `Tensor` (plus some extra arguments) to functions that accept a `Builder` (plus some extra arguments). Due to this, not all methods will work as expected, an obvious example is [tf.placeholder](https://www.tensorflow.org/versions/r0.9/api_docs/python/io_ops.html#placeholder), this function is automatically included but it doesn't take a Tensor as its first parameter so it doesn't make any sense a method of this class. Right now the current policy of which of these functions are include/exclude is a blacklist approach so that only functions that are known to cause serious problems (like having the same name as basic methods) are excluded and all the functions you are likely going to use are included. 91 | 3. Based on point 1 and 2, the next set of function are defined as: if `f` is a function in `tf` or `tf.nn` with name `fname`, then the method `fname_layer` exists in this class. These methods use `fully_connected` and `f` to create a layer with `f` as its activation function. While you don't REALLY need them, `.softmax_layer(5)` reads much better than `.fully_connected(5, activation_fn=tf.nn.softmax)`. 92 | 93 | #### Using the methods 94 | 95 | So we used the methods `.sigmoid_layer(2)` and `.linear_layer(1)` to create our `logit`. Now to create the `activation` function (rather Tensor) of our network we did the following 96 | 97 | ```python 98 | activation = ( 99 | logit 100 | .sigmoid() 101 | .tensor() 102 | ) 103 | ``` 104 | 105 | This was basically just applying `tf.sigmoid` over the `logit`. The method `.tensor` allows us to actually get back the `Tensor` inside the Builder. 106 | 107 | #### The map method 108 | 109 | Finally we created our network `trainer` doing the following 110 | 111 | ```python 112 | trainer = ( 113 | logit 114 | .sigmoid_cross_entropy_with_logits(y) # loss 115 | .map(tf.train.AdamOptimizer(0.01).minimize) 116 | .tensor() 117 | ) 118 | ``` 119 | 120 | Initially we just indirectly applyed the function `tf.nn.sigmoid_cross_entropy_with_logits` over the `login` and the target's placeholder `y`, to get out our *loss* Tensor. But then we used a custom method from the `Builder` class: [map](http://cgarciae.github.io/phi/core/index.html#phi.core.BuilderBase.map). 121 | 122 | `map` takes any function that accepts a Tensor as its first parameter (and some extra arguments), applies that function to the Tensor inside our Builder (plus the extra arguments), and returns a Builder with the new Tensor. In this case our function was the *unbounded method* `minimize` of the `AdamOptimizer` instace (created in-line) that expect a loss Tensor and returns a Tensor that performs the computation that trains our network. 123 | 124 | The thing is, given that we have `map` we actually don't REALLY need most of the other methods! We could e.g. have written the initial structure of our network like this 125 | 126 | ```python 127 | logit = ( 128 | tb 129 | .build(x) 130 | .map(tf.contrib.layers.fully_connected, 2, activation_fn=tf.nn.sigmoid) 131 | .map(tf.contrib.layers.fully_connected, 1, activation_fn=None) 132 | ) 133 | ``` 134 | 135 | instead of 136 | 137 | ```python 138 | logit = ( 139 | tb 140 | .build(x) 141 | .sigmoid_layer(2) 142 | .linear_layer(1) 143 | ) 144 | ``` 145 | 146 | but as you see the latter is more compact and readable. The important thing is that you understand that you can use `map` to incorporate functions not registered in the Builder class naturally into the computation. 147 | 148 | ## Training 149 | Finally, given that we have constructed the `trainer` and `activation` Tensors, lets use regular TensorFlow operations to trainer the network. We will train for 2000 epochs using full batch training (given that we only have 4 training examples) and then print out the prediction for each case of the XOR using the `activation` Tensor. 150 | 151 | ```python 152 | # create session 153 | sess = tf.Session() 154 | sess.run(tf.initialize_all_variables()) 155 | 156 | # train 157 | for i in range(2000): 158 | sess.run(trainer, feed_dict={x: X, y: Y}) 159 | 160 | # test 161 | for i in range(len(X)): 162 | print "{0} ==> {1}".format(X[i], sess.run(activation, feed_dict={x: X[i:i+1,:]})) 163 | ``` 164 | 165 | Congratulations! You have just solved the XOR problem using TensorBuilder. Not much of a feat for a serious Machine Learning Engineer, but you have the basic knowledge of the TensorBuilder API. 166 | 167 | ## What's Next? 168 | In the next chapters you will learn how to create branched neural networks (important in many architectures), use scoping mechanisms to specify some attributes about the Tensor we build, and explore the Domain Specific Language (DSL) using all the previous knowledge to enable you to code even faster. 169 | 170 | -------------------------------------------------------------------------------- /phi/tests/test_builders.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | from __future__ import unicode_literals 5 | 6 | from phi.api import * 7 | import math 8 | import pytest 9 | # from phi import tb 10 | 11 | add2 = P + 2 12 | mul3 = P * 3 13 | get_list = lambda x: [1,2,3] 14 | a2_plus_b_minus_2c = lambda a, b, c: a ** 2 + b - 2*c 15 | 16 | 17 | @P.Register("test.lib") 18 | def add(a, b): 19 | """Some docs""" 20 | return a + b 21 | 22 | @P.Register2("test.lib") 23 | def pow(a, b): 24 | return a ** b 25 | 26 | @P.RegisterMethod("test.lib") 27 | def give_me_1000(self): 28 | return 1000 29 | 30 | class DummyContext: 31 | def __init__(self, value): 32 | self.value = value 33 | 34 | def __enter__(self): 35 | return self.value 36 | def __exit__(self, type, value, traceback): 37 | pass 38 | 39 | 40 | class TestBuilder(object): 41 | """docstring for TestBuilder""" 42 | 43 | @classmethod 44 | def setup_method(self): 45 | pass 46 | 47 | def test_C_1(self): 48 | assert P.Then(add2)(4) == 6 49 | assert P.Then(add2).Then(mul3)(4) == 18 50 | 51 | 52 | def test_patch_at(self): 53 | from . import some_module 54 | 55 | P.PatchAt(1, some_module, whitelist_predicate=lambda name: "some" in name) 56 | P.PatchAt(2, some_module, whitelist_predicate=lambda name: "other" in name) 57 | 58 | ################### 59 | 60 | f = P.some_fun() >> Obj.lower() 61 | 62 | assert f(None) == "yes" 63 | 64 | ######################## 65 | 66 | f = P.other_fun(6.0) >> P + 1 67 | 68 | assert f(2.0) == 4 69 | 70 | def test_methods(self): 71 | x = P.Pipe( 72 | "hello world", 73 | Obj.split(" ") 74 | .filter(P.Contains("w").Not()) 75 | .map(len) 76 | .First() 77 | ) 78 | 79 | assert not P.Pipe( 80 | [1,2,3], 81 | P.Contains(5) 82 | ) 83 | 84 | class A(object): 85 | def something(self, x): 86 | return "y" * x 87 | 88 | 89 | assert "yyy" == P.Pipe( 90 | A(), 91 | P.Obj.something(3) #used something 92 | ) 93 | 94 | def test_rrshift(self): 95 | builder = P.Seq( 96 | P + 1, 97 | P * 2, 98 | P + 4 99 | ) 100 | 101 | assert 10 == 2 >> builder 102 | 103 | def test_compose(self): 104 | f = P.Seq( 105 | P + 1, 106 | P * 2, 107 | P + 4 108 | ) 109 | 110 | assert 10 == f(2) 111 | 112 | 113 | def test_compose_list(self): 114 | f = P.Seq( 115 | P + 1, 116 | Write( x = P * 2), 117 | P + 4, 118 | List( 119 | P + 2 120 | , 121 | P / 2 122 | , 123 | Read('x') 124 | ) 125 | ) 126 | 127 | assert [12, 5, 6] == f(2) 128 | 129 | f = P.Seq( 130 | P + 1, 131 | P * 2, 132 | P + 4, 133 | List( 134 | Write(x = P + 2) 135 | , 136 | P / 2 137 | , 138 | Read('x') 139 | ) 140 | ) 141 | 142 | assert [12, 5, 12] == f(2) 143 | 144 | def test_compose_list_reduce(self): 145 | f = P.Seq( 146 | P + 1, 147 | P * 2, 148 | P + 4, 149 | List( 150 | P + 2 151 | , 152 | P / 2 153 | ), 154 | sum 155 | ) 156 | 157 | assert 17 == f(2) 158 | 159 | def test_random(self): 160 | 161 | assert 9 == P.Pipe( 162 | "Hola Cesar", 163 | P.Obj.split(" "), 164 | P.map(len) 165 | .sum() 166 | ) 167 | 168 | def test_0(self): 169 | from datetime import datetime 170 | import time 171 | 172 | t0 = datetime.now() 173 | 174 | time.sleep(0.01) 175 | 176 | t1 = 2 >> P.Seq( 177 | P + 1, 178 | P.Then0(datetime.now) 179 | ) 180 | 181 | assert t1 > t0 182 | 183 | def test_1(self): 184 | assert 9 == 2 >> P.Seq( 185 | P + 1, 186 | P.Then(math.pow, 2) 187 | ) 188 | 189 | def test_2(self): 190 | assert [2, 4] == [1, 2, 3] >> P.Seq( 191 | P 192 | .Then2(map, P + 1) 193 | .Then2(filter, P % 2 == 0) 194 | .list() #list only needed in Python 3 195 | ) 196 | 197 | assert [2, 4] == P.Pipe( 198 | [1, 2, 3], 199 | P 200 | .Then2(map, P + 1) 201 | .Then2(filter, P % 2 == 0) 202 | .list() #list only needed in Python 3 203 | ) 204 | 205 | 206 | def test_underscores(self): 207 | assert P.Then(a2_plus_b_minus_2c, 2, 4)(3) == 3 # (3)^2 + 2 - 2*4 208 | assert P.Then2(a2_plus_b_minus_2c, 2, 4)(3) == -1 # (2)^2 + 3 - 2*4 209 | assert P.Then3(a2_plus_b_minus_2c, 2, 4)(3) == 2 # (2)^2 + 4 - 2*3 210 | 211 | def test_pipe(self): 212 | assert P.Pipe(4, add2, mul3) == 18 213 | 214 | assert [18, 14] == P.Pipe( 215 | 4, 216 | List( 217 | Seq( 218 | add2, 219 | mul3 220 | ) 221 | , 222 | Seq( 223 | mul3, 224 | add2 225 | ) 226 | ) 227 | ) 228 | 229 | assert [18, 18, 15, 16] == P.Pipe( 230 | 4, 231 | List( 232 | Seq( 233 | add2, 234 | mul3 235 | ) 236 | , 237 | List( 238 | Seq( 239 | add2, 240 | mul3 241 | ) 242 | , 243 | Seq( 244 | mul3, 245 | add2, 246 | List( 247 | P + 1, 248 | P + 2 249 | ) 250 | ) 251 | ) 252 | ) 253 | .Flatten() 254 | ) 255 | 256 | 257 | assert [18, [18, 14, get_list(None)]] == P.Pipe( 258 | 4, 259 | List( 260 | Seq( 261 | add2, 262 | mul3 263 | ) 264 | , 265 | List( 266 | Seq( 267 | add2, 268 | mul3 269 | ) 270 | , 271 | Seq( 272 | mul3, 273 | add2 274 | ) 275 | , 276 | get_list 277 | ) 278 | ) 279 | ) 280 | 281 | [a, [b, c]] = P.Pipe( 282 | 4, 283 | List( 284 | Seq( 285 | add2, 286 | mul3 287 | ) 288 | , 289 | List( 290 | Seq( 291 | add2, 292 | mul3 293 | ) 294 | , 295 | Seq( 296 | mul3, 297 | add2 298 | ) 299 | ) 300 | ) 301 | ) 302 | 303 | assert a == 18 and b == 18 and c == 14 304 | 305 | def test_context(self): 306 | y = P.Ref('y') 307 | 308 | length = P.Pipe( 309 | "phi/tests/test.txt", 310 | P.With( open, 311 | Context, 312 | Obj.read(), y.write, 313 | len 314 | ) 315 | ) 316 | 317 | assert length == 11 318 | assert y() == "hello world" 319 | 320 | def test_register_1(self): 321 | 322 | #register 323 | assert 5 == P.Pipe( 324 | 3, 325 | P.add(2) 326 | ) 327 | 328 | #Register2 329 | assert 8 == P.Pipe( 330 | 3, 331 | P.pow(2) 332 | ) 333 | 334 | #RegisterMethod 335 | assert P.give_me_1000() == 1000 336 | 337 | def test_reference(self): 338 | add_ref = P.Ref('add_ref') 339 | 340 | assert 8 == P.Pipe(3 >> P.add(2) >> add_ref.write >> P.add(3)) 341 | assert 5 == add_ref() 342 | 343 | def test_ref_props(self): 344 | 345 | a = P.Ref('a') 346 | b = P.Ref('b') 347 | 348 | assert [7, 3, 5] == P.Pipe( 349 | 1, 350 | add2, a.write, 351 | add2, b.write, 352 | add2, 353 | List( 354 | Seq(), 355 | a, 356 | b 357 | ) 358 | ) 359 | 360 | def test_scope_property(self): 361 | 362 | assert "some random text" == P.Pipe( 363 | "some ", 364 | P.With( DummyContext("random "), 365 | P + P.Context, 366 | P.With( DummyContext("text"), 367 | P + P.Context 368 | ) 369 | ) 370 | ) 371 | 372 | with pytest.raises(Exception): 373 | P.Context() #Cannot use it outside of With 374 | 375 | def test_ref_integraton_with_dsl(self): 376 | 377 | y = P.Ref('y') 378 | 379 | 380 | assert 5 == P.Pipe( 381 | 1, 382 | P + 4, 383 | y.write, 384 | P * 10, 385 | y 386 | ) 387 | 388 | assert 5 == P.Pipe( 389 | 1, 390 | P + 4, 391 | y.write, 392 | P * 10, 393 | y 394 | ) 395 | 396 | assert 5 == P.Pipe( 397 | 1, 398 | Write(y = P + 4), 399 | P * 10, 400 | Read.y 401 | ) 402 | 403 | def test_list(self): 404 | 405 | assert [['4', '6'], [4, 6]] == P.Pipe( 406 | 3, 407 | List( 408 | P + 1 409 | , 410 | P * 2 411 | ), 412 | List( 413 | P.Then2(map, str).list() #list only needed in Python 3 414 | , 415 | Seq() 416 | ) 417 | ) 418 | 419 | def test_read_method(self): 420 | 421 | assert 1 == P.Pipe( 422 | 1, 423 | Write(s = P + 1), 424 | P * 100, 425 | Read('s'), P - 1 426 | ) 427 | 428 | assert 1 == P.Pipe( 429 | 1, 430 | Write(s = P + 1), 431 | P * 100, 432 | Read('s') - 1 433 | ) 434 | 435 | assert 1 == P.Pipe( 436 | 1, 437 | Write(s = P + 1), 438 | P * 100, 439 | Read.s - 1 440 | ) 441 | 442 | assert 1 == P.Pipe( 443 | 1, 444 | Write(s = P + 1), 445 | P * 100, 446 | Read('s') - 1 447 | ) 448 | 449 | assert 1 == P.Pipe( 450 | 1, 451 | Write(s = P + 1), 452 | P * 100, 453 | Read.s - 1 454 | ) 455 | 456 | assert 1 == P.Pipe( 457 | 1, 458 | Write(s = P + 1), 459 | P * 100, 460 | Read.s - 1 461 | ) 462 | 463 | assert 1 == P.Pipe( 464 | 1, 465 | Write(s = P + 1), 466 | P * 100, 467 | Read.s - 1 468 | ) 469 | -------------------------------------------------------------------------------- /phi/tests/test_examples.py: -------------------------------------------------------------------------------- 1 | from phi.api import * 2 | 3 | class TestExamples(object): 4 | """docstring for TestExamples.""" 5 | 6 | def test_example_1(self): 7 | 8 | text = "a bb ccc" 9 | avg_word_length = P.Pipe( 10 | text, 11 | Obj.split(" "), #['a', 'bb', 'ccc'] 12 | P.map(len), #[1, 2, 3] 13 | list, # python 3 only 14 | P.sum() / len #6 / 3 == 2 15 | ) 16 | 17 | assert 2 == avg_word_length 18 | 19 | 20 | text = "a bb ccc" 21 | avg_word_length = P.Pipe( 22 | text, 23 | Obj.split(" "), #['a', 'bb', 'ccc'] 24 | P.map(len), #[1, 2, 3] 25 | list, # python 3 only 26 | Seq(sum) / len #6 / 3 == 2 27 | ) 28 | 29 | assert 2 == avg_word_length 30 | 31 | 32 | text = "a bb ccc" 33 | avg_word_length = P.Pipe( 34 | text, 35 | Obj.split(" "), #['a', 'bb', 'ccc'] 36 | P.map(len), #[1, 2, 3] 37 | list, # python 3 only 38 | P.sum() / P.len() #6 / 3 == 2 39 | ) 40 | 41 | assert 2 == avg_word_length 42 | 43 | 44 | avg_word_length = P.Pipe( 45 | "1 22 333", 46 | Obj.split(' '), # ['1', '22', '333'] 47 | P.map(len), # [1, 2, 3] 48 | list, # python 3 only 49 | List( 50 | sum # 1 + 2 + 3 == 6 51 | , 52 | len # len([1, 2, 3]) == 3 53 | ), 54 | P[0] / P[1] # sum / len == 6 / 3 == 2 55 | ) 56 | 57 | assert avg_word_length == 2 58 | 59 | def test_getting_started(self): 60 | from phi import P 61 | 62 | def add1(x): return x + 1 63 | def mul3(x): return x * 3 64 | 65 | x = P.Pipe( 66 | 1.0, #input 1 67 | add1, #1 + 1 == 2 68 | mul3 #2 * 3 == 6 69 | ) 70 | 71 | assert x == 6 72 | 73 | ################ 74 | ################ 75 | 76 | from phi import P 77 | 78 | x = P.Pipe( 79 | 1.0, #input 1 80 | P + 1, #1 + 1 == 2 81 | P * 3 #2 * 3 == 6 82 | ) 83 | 84 | assert x == 6 85 | 86 | ################ 87 | ################ 88 | 89 | from phi import P, List 90 | 91 | [x, y] = P.Pipe( 92 | 1.0, #input 1 93 | List( 94 | P + 1 #1 + 1 == 2 95 | , 96 | P * 3 #1 * 3 == 3 97 | ) 98 | ) 99 | 100 | assert x == 2 101 | assert y == 3 102 | 103 | ################ 104 | ################ 105 | 106 | from phi import P, List 107 | 108 | [x, y] = P.Pipe( 109 | 1.0, #input 1 110 | P * 2, #1 * 2 == 2 111 | List( 112 | P + 1 #2 + 1 == 3 113 | , 114 | P * 3 #2 * 3 == 6 115 | ) 116 | ) 117 | 118 | assert x == 3 119 | assert y == 6 120 | 121 | ################ 122 | ################ 123 | 124 | from phi import P, Rec 125 | 126 | result = P.Pipe( 127 | 1.0, #input 1 128 | P * 2, #1 * 2 == 2 129 | Dict( 130 | x = P + 1 #2 + 1 == 3 131 | , 132 | y = P * 3 #2 * 3 == 6 133 | ) 134 | ) 135 | 136 | assert result.x == 3 137 | assert result.y == 6 138 | 139 | ################ 140 | ################ 141 | 142 | from phi import P, Rec 143 | 144 | result = P.Pipe( 145 | 1.0, #input 1 146 | P * 2, #1 * 2 == 2 147 | Dict( 148 | x = P + 1 #2 + 1 == 3 149 | , 150 | y = P * 3 #2 * 3 == 6 151 | ), 152 | Rec.x / Rec.y #3 / 6 == 0.5 153 | ) 154 | 155 | assert result == 0.5 156 | 157 | ################ 158 | ################ 159 | 160 | from phi import P, Rec, List, Write, Read 161 | 162 | [result, s] = P.Pipe( 163 | 1.0, #input 1 164 | Write(s = P * 2), #s = 2 * 1 == 2 165 | Dict( 166 | x = P + 1 #2 + 1 == 3 167 | , 168 | y = P * 3 #2 * 3 == 6 169 | ), 170 | List( 171 | Rec.x / Rec.y #3 / 6 == 0.5 172 | , 173 | Read('s') #load 's' == 2 174 | ) 175 | ) 176 | 177 | assert result == 0.5 178 | assert s == 2 179 | 180 | ################ 181 | ################ 182 | 183 | from phi import P, Rec, List, Write, Read 184 | 185 | [result, s] = P.Pipe( 186 | 1.0, #input 1 187 | P * 2, Write('s'), #s = 2 * 1 == 2 188 | Dict( 189 | x = P + 1 #2 + 1 == 3 190 | , 191 | y = P * 3 #2 * 3 == 6 192 | ), 193 | List( 194 | Rec.x / Rec.y #3 / 6 == 0.5 195 | , 196 | Read('s') #load 's' == 2 197 | ) 198 | ) 199 | 200 | assert result == 0.5 201 | assert s == 2 202 | 203 | ################ 204 | ################ 205 | 206 | from phi import P, Rec, Write, Read, List 207 | 208 | [result, s] = P.Pipe( 209 | 1.0, #input 1 210 | Write(s = P * 2), #s = 2 * 1 == 2 211 | Dict( 212 | x = P + 1 #2 + 1 == 3 213 | , 214 | y = P * 3 #2 * 3 == 6 215 | ), 216 | List( 217 | Rec.x / Rec.y #3 / 6 == 0.5 218 | , 219 | Read.s + 3 # 2 + 3 == 5 220 | ) 221 | ) 222 | 223 | assert result == 0.5 224 | assert s == 5 225 | 226 | ################ 227 | ################ 228 | 229 | from phi import P, Rec, Read, Write 230 | 231 | [result, s] = P.Pipe( 232 | 1.0, #input 1 233 | Write(s = P * 2), #s = 2 * 1 == 2 234 | Dict( 235 | x = P + 1 #2 + 1 == 3 236 | , 237 | y = P * 3 #2 * 3 == 6 238 | ), 239 | List( 240 | Rec.x / Rec.y #3 / 6 == 0.5 241 | , 242 | Read.s + 3 # 2 + 3 == 5 243 | ) 244 | ) 245 | 246 | assert result == 0.5 247 | assert s == 5 248 | 249 | ################ 250 | ################ 251 | 252 | from phi import P, Rec, Val 253 | 254 | [result, s, val] = P.Pipe( 255 | 1.0, #input 1 256 | Write(s = P * 2), #s = 2 * 1 == 2 257 | Dict( 258 | x = P + 1 #2 + 1 == 3 259 | , 260 | y = P * 3 #2 * 3 == 6 261 | ), 262 | List( 263 | Rec.x / Rec.y #3 / 6 == 0.5 264 | , 265 | Read.s + 3 # 2 + 3 == 5 266 | , 267 | Val(9) + 1 #input 9 and add 1, gives 10 268 | ) 269 | ) 270 | 271 | assert result == 0.5 272 | assert s == 5 273 | assert val == 10 274 | 275 | ######################### 276 | ######################### 277 | 278 | from phi import P, Rec, Read, Write, Val, If 279 | 280 | [result, s, val] = P.Pipe( 281 | 1.0, #input 1 282 | Write(s = (P + 3) / (P + 1)), #s = 4 / 2 == 2 283 | Dict( 284 | x = P + 1 #2 + 1 == 3 285 | , 286 | y = P * 3 #2 * 3 == 6 287 | ), 288 | List( 289 | Rec.x / Rec.y #3 / 6 == 0.5 290 | , 291 | Read.s + 3 # 2 + 3 == 5 292 | , 293 | If( Rec.y > 7, 294 | Val(9) + 1 #input 9 and add 1, gives 10 295 | ).Elif( Rec.y < 4, 296 | "Yes" 297 | ).Else( 298 | "Sorry, come back latter." 299 | ) 300 | ) 301 | ) 302 | 303 | assert result == 0.5 304 | assert s == 5 305 | assert val == "Sorry, come back latter." 306 | 307 | 308 | ###################################### 309 | ####################################### 310 | 311 | from phi import P, Rec, Read, Write, Val, If 312 | 313 | f = P.Seq( 314 | Write(s = (P + 3) / (P + 1)), #s = 4 / 2 == 2 315 | Dict( 316 | x = P + 1 #2 + 1 == 3 317 | , 318 | y = P * 3 #2 * 3 == 6 319 | ), 320 | List( 321 | Rec.x / Rec.y #3 / 6 == 0.5 322 | , 323 | Read.s + 3 # 2 + 3 == 5 324 | , 325 | If( Rec.y > 7, 326 | Val(9) + 1 #input 9 and add 1, gives 10 327 | ).Else( 328 | "Sorry, come back latter." 329 | ) 330 | ) 331 | ) 332 | 333 | [result, s, val] = f(1.0) 334 | 335 | assert result == 0.5 336 | assert s == 5 337 | assert val == "Sorry, come back latter." 338 | 339 | 340 | 341 | def test_builder_MakeRefContext(self): 342 | 343 | from phi import P 344 | 345 | assert 2 == P.Pipe( 346 | Write(s = 1), #s = 1 347 | P.Seq( 348 | Write(s = P + 1), #s = 2 349 | ), 350 | Read('s') # s == 2 351 | ) 352 | 353 | ################################ 354 | ################################ 355 | 356 | 357 | 358 | 359 | def test_builder_NPipe(self): 360 | 361 | from phi import P 362 | 363 | assert 1 == P.Pipe( 364 | Write(s = 1), # write s == 1, outer context 365 | lambda x: P.Pipe( 366 | x, 367 | Write(s = P + 1) # write s == 2, inner context 368 | ), 369 | Read('s') # read s == 1, outer context 370 | ) 371 | 372 | ############################# 373 | ############################# 374 | 375 | def test_not(self): 376 | 377 | from phi import P 378 | 379 | assert True == P.Pipe( 380 | 1, 381 | P + 1, # 1 + 1 == 2 382 | P > 5, # 2 > 5 == False 383 | P.Not() # not False == True 384 | ) 385 | 386 | ################################ 387 | ################################ 388 | 389 | from phi import P 390 | 391 | assert True == P.Pipe( 392 | 1, 393 | (P + 1 > 5).Not() # not 1 + 1 > 5 == not 2 > 5 == not False == True 394 | ) 395 | 396 | ############################ 397 | ############################# 398 | 399 | from phi import P 400 | 401 | f = (P + 1 > 5).Not() #lambda x: not x + 1 > 5 402 | 403 | assert f(1) == True 404 | 405 | def test_contains(self): 406 | 407 | from phi import P 408 | 409 | assert False == P.Pipe( 410 | [1,2,3,4], 411 | P.filter(P % 2 != 0) #[1, 3], keeps odds 412 | .Contains(4) #4 in [1, 3] == False 413 | ) 414 | 415 | def test_ref(self): 416 | 417 | from phi import P, Obj, Ref 418 | 419 | assert {'a': 97, 'b': 98, 'c': 99} == P.Pipe( 420 | "a b c", Obj 421 | .split(' ') #['a', 'b', 'c'] 422 | .Write(keys = P) # key = ['a', 'b', 'c'] 423 | .map(ord), # [ord('a'), ord('b'), ord('c')] == [97, 98, 99] 424 | lambda it: zip(Ref.keys, it), # [('a', 97), ('b', 98), ('c', 99)] 425 | dict # {'a': 97, 'b': 98, 'c': 99} 426 | ) 427 | 428 | def test_if(self): 429 | 430 | from phi import P, Val 431 | 432 | assert "Between 2 and 10" == P.Pipe( 433 | 5, 434 | P.If(P > 10, 435 | "Greater than 10" 436 | ).Elif(P < 2, 437 | "Less than 2" 438 | ).Else( 439 | "Between 2 and 10" 440 | ) 441 | ) 442 | 443 | def test_pipe_branch(self): 444 | 445 | assert [11, 12] == 10 >> List( P + 1, P + 2) 446 | 447 | 448 | def test_state(self): 449 | 450 | f = Read("a") + 5 >> Write(a = P) 451 | assert f(None, True, a=0) == (5, {"a": 5}) 452 | 453 | 454 | f = Read.a + 5 >> Write(a = P) 455 | assert f(None, True, a=0) == (5, {"a": 5}) 456 | 457 | 458 | def test_math(self): 459 | import math 460 | 461 | f = P.map(P ** 2) >> list >> P[0] + P[1] >> math.sqrt 462 | 463 | assert f([3, 4]) == 5 464 | 465 | 466 | def test_operators(self): 467 | 468 | f = (P * 6) / (P + 2) 469 | 470 | assert f(2) == 3 # (2 * 6) / (2 + 2) == 12 / 4 == 3 471 | 472 | ########################### 473 | 474 | def test_get_item(self): 475 | 476 | f = P[0] + P[-1] #add the first and last elements 477 | 478 | assert f([1,2,3,4]) == 5 #1 + 4 == 5 479 | 480 | ################## 481 | 482 | def test_field_access(self): 483 | 484 | from collections import namedtuple 485 | Point = namedtuple('Point', ['x', 'y']) 486 | 487 | f = Rec.x + Rec.y #add the x and y fields 488 | 489 | assert f(Point(3, 4)) == 7 #point.x + point.y == 3 + 4 == 7 490 | 491 | def test_method_calling(self): 492 | 493 | f = Obj.upper() + ", " + Obj.lower() #lambda s: s.upper() + ", " + s.lower() 494 | 495 | assert f("HEllo") == "HELLO, hello" # "HEllo".upper() + ", " + "HEllo".lower() == "HELLO" + ", " + "hello" == "HELLO, hello" 496 | 497 | ############################# 498 | 499 | def test_rshif_and_lshift(arg): 500 | import math 501 | 502 | f = P + 7 >> math.sqrt #executes left to right 503 | 504 | assert f(2) == 3 # math.sqrt(2 + 7) == math.sqrt(9) == 3 505 | 506 | ################ 507 | 508 | f = math.sqrt << P + 7 #executes right to left 509 | 510 | assert f(2) == 3 # math.sqrt(2 + 7) == math.sqrt(9) == 3 511 | 512 | ####################### 513 | 514 | def test_seq_and_pipe(arg): 515 | import math 516 | 517 | f = Seq( 518 | str, 519 | P + "00", 520 | int, 521 | math.sqrt 522 | ) 523 | 524 | assert f(1) == 10 # sqrt(int("1" + "00")) == sqrt(100) == 10 525 | 526 | ################################## 527 | 528 | assert 10 == Pipe( 529 | 1, #input 530 | str, # "1" 531 | P + "00", # "1" + "00" == "100" 532 | int, # 100 533 | math.sqrt #sqrt(100) == 10 534 | ) 535 | 536 | ###################################### 537 | 538 | 539 | def test_list_tuple_ect(arg): 540 | f = List( P + 1, P * 10 ) #lambda x: [ x +1, x * 10 ] 541 | 542 | assert f(3) == [ 4, 30 ] # [ 3 + 1, 3 * 10 ] == [ 4, 30 ] 543 | 544 | 545 | ################################## 546 | 547 | f = Dict( x = P + 1, y = P * 10 ) #lambda x: [ x +1, x * 10 ] 548 | 549 | d = f(3) 550 | 551 | assert d == { 'x': 4, 'y': 30 } # { 'x': 3 + 1, 'y': 3 * 10 } == { 'x': 4, 'y': 30 } 552 | assert d.x == 4 #access d['x'] via field access as d.x 553 | assert d.y == 30 #access d['y'] via field access as d.y 554 | 555 | ######################################### 556 | 557 | 558 | def test_state_read_write(arg): 559 | assert [70, 30] == Pipe( 560 | 3, 561 | Write(s = P * 10), #s = 3 * 10 == 30 562 | P + 5, #30 + 5 == 35 563 | List( 564 | P * 2 # 35 * 2 == 70 565 | , 566 | Read('s') #s == 30 567 | ) 568 | ) 569 | 570 | ########################### 571 | 572 | 573 | def test_thens(arg): 574 | def repeat_word(word, times, upper=False): 575 | if upper: 576 | word = word.upper() 577 | 578 | return [ word ] * times 579 | 580 | f = P[::-1] >> Then(repeat_word, 3) 581 | g = P[::-1] >> Then(repeat_word, 3, upper=True) 582 | 583 | assert f("ward") == ["draw", "draw", "draw"] 584 | assert g("ward") == ["DRAW", "DRAW", "DRAW"] 585 | 586 | ########################### 587 | 588 | # since map and filter receive the iterable on their second argument, you have to use `Then2` 589 | f = Then2(filter, P % 2 == 0) >> Then2(map, P**2) >> list #lambda x: map(lambda z: z**2, filter(lambda z: z % 2 == 0, x)) 590 | 591 | assert f([1,2,3,4,5]) == [4, 16] #[2**2, 4**2] == [4, 16] 592 | 593 | 594 | ###################################### 595 | 596 | f = P.filter(P % 2 == 0) >> P.map(P**2) >> list #lambda x: map(lambda z: z**2, filter(lambda z: z % 2 == 0, x)) 597 | 598 | assert f([1,2,3,4,5]) == [4, 16] #[2**2, 4**2] == [4, 16] 599 | 600 | 601 | ###################################### 602 | 603 | def test_val(self): 604 | 605 | f = Val(42) #lambda x: 42 606 | 607 | assert f("whatever") == 42 608 | 609 | ##################################### 610 | 611 | def test_others(self): 612 | f = Obj.split(' ') >> P.map(len) >> sum >> If( (P < 15).Not(), "Great! Got {0} letters!".format).Else("Too short, need at-least 15 letters") 613 | 614 | assert f("short frase") == "Too short, need at-least 15 letters" 615 | assert f("some longer frase") == "Great! Got 15 letters!" 616 | 617 | ########################################### 618 | 619 | def test_dsl(self): 620 | f = P**2 >> List( P, Val(3), Val(4) ) #lambda x: [ x**2] 621 | 622 | assert f(10) == [ 100, 3, 4 ] # [ 10**2, 3, 4 ] == [ 100, 3, 4 ] 623 | 624 | ############################################ 625 | 626 | f = P**2 >> List( P, 3, 4 ) 627 | 628 | assert f(10) == [ 100, 3, 4 ] # [ 10 ** 2, 3, 4 ] == [ 100, 3, 4 ] 629 | 630 | 631 | ########################################### 632 | 633 | f = P**2 >> [ P, 3, 4 ] 634 | 635 | assert f(10) == [ 100, 3, 4 ] # [ 10 ** 2, 3, 4 ] == [ 100, 3, 4 ] 636 | 637 | 638 | ############################################ 639 | 640 | assert [ 100, 3, 4 ] == Pipe( 641 | 10, 642 | P**2, # 10**2 == 100 643 | [ P, 3, 4 ] #[ 100, 3, 4 ] 644 | ) 645 | 646 | 647 | def test_f(self): 648 | 649 | f = F((P + "!!!", 42, Obj.upper())) #Tuple(P + "!!!", Val(42), Obj.upper()) 650 | 651 | assert f("some tuple") == ("some tuple!!!", 42, "SOME TUPLE") 652 | 653 | ############################################# 654 | 655 | f = F([ P + n for n in range(5) ]) >> [ len, sum ] # lambda x: [ len([ x, x+1, x+2, x+3, x+4]), sum([ x, x+1, x+2, x+3, x+4]) ] 656 | 657 | assert f(10) == [ 5, 60 ] # [ len([10, 11, 12, 13, 14]), sum([10, 11, 12, 13, 14])] == [ 5, (50 + 0 + 1 + 2 + 3 + 4) ] == [ 5, 60 ] 658 | 659 | def test_fluent(self): 660 | 661 | f = Dict( 662 | x = 2 * P, 663 | y = P + 1 664 | ).Tuple( 665 | Rec.x + Rec.y, 666 | Rec.y / Rec.x 667 | ) 668 | 669 | assert f(1) == (4, 1) # ( x + y, y / x) == ( 2 + 2, 2 / 2) == ( 4, 1 ) 670 | 671 | ################################# 672 | 673 | f = Obj.split(' ') >> P.map(len) >> sum >> If( (P < 15).Not(), "Great! Got {0} letters!".format).Else("Too short, need at-least 15 letters") 674 | 675 | assert f("short frase") == "Too short, need at-least 15 letters" 676 | assert f("some longer frase") == "Great! Got 15 letters!" 677 | 678 | ###################################################### 679 | 680 | f = ( 681 | Obj.split(' ') 682 | .map(len) 683 | .sum() 684 | .If( (P < 15).Not(), 685 | "Great! Got {0} letters!".format 686 | ).Else( 687 | "Too short, need at-least 15 letters" 688 | ) 689 | ) 690 | 691 | assert f("short frase") == "Too short, need at-least 15 letters" 692 | assert f("some longer frase") == "Great! Got 15 letters!" 693 | 694 | ########################################################### 695 | 696 | def test_Register(self): 697 | 698 | from phi import PythonBuilder 699 | 700 | class MyBuilder(PythonBuilder): 701 | pass 702 | 703 | M = MyBuilder() 704 | 705 | @MyBuilder.Register("my.lib.") 706 | def remove_longer_than(some_list, n): 707 | return [ elem for elem in some_list if len(elem) <= n ] 708 | 709 | f = Obj.lower() >> Obj.split(' ') >> M.remove_longer_than(6) 710 | 711 | assert f("SoMe aRe LONGGGGGGGGG") == ["some", "are"] 712 | 713 | ####################################################### 714 | 715 | -------------------------------------------------------------------------------- /phi/README-template.md: -------------------------------------------------------------------------------- 1 | # Phi 2 | Phi library for functional programming in Python that intends to remove as much of the pain as possible from your functional programming experience in Python. 3 | 4 | ## Import 5 | For demonstration purposes we will import right now everything we will need for the rest of the exercises like this 6 | ```python 7 | from phi.api import * 8 | ``` 9 | but you can also import just what you need from the `phi` module. 10 | 11 | ## Math-like Lambdas 12 | 13 | #### Operators 14 | 15 | Using the `P` object you can create quick lambdas using any operator. You can write things like 16 | 17 | ```python 18 | f = (P * 6) / (P + 2) #lambda x: (x * 6) / (x + 2) 19 | 20 | assert f(2) == 3 # (2 * 6) / (2 + 2) == 12 / 4 == 3 21 | ``` 22 | 23 | where the expression for `f` is equivalent to 24 | 25 | ```python 26 | f = lambda x: (x * 6) / (x + 2) 27 | ``` 28 | 29 | #### getitem 30 | You can also use the `P` object to create lambdas that access the items of a collection 31 | ```python 32 | f = P[0] + P[-1] #lambda x: x[0] + x[-1] 33 | 34 | assert f([1,2,3,4]) == 5 #1 + 4 == 5 35 | ``` 36 | 37 | #### field access 38 | If you want create lambdas that access the field of some entity you can use the `Rec` (for Record) object an call that field on it 39 | ```python 40 | from collections import namedtuple 41 | Point = namedtuple('Point', ['x', 'y']) 42 | 43 | f = Rec.x + Rec.y #lambda p: p.x + p.y 44 | 45 | assert f(Point(3, 4)) == 7 #point.x + point.y == 3 + 4 == 7 46 | ``` 47 | #### method calling 48 | If you want to create a lambda that calls the method of an object you use the `Obj` object and call that method on it with the parameters 49 | ```python 50 | f = Obj.upper() + ", " + Obj.lower() #lambda s: s.upper() + ", " + s.lower() 51 | 52 | assert f("HEllo") == "HELLO, hello" # "HEllo".upper() + ", " + "HEllo".lower() == "HELLO" + ", " + "hello" == "HELLO, hello" 53 | ``` 54 | Here no parameters were needed but in general 55 | ```python 56 | f = Obj.some_method(arg1, arg2, ...) #lambda obj: obj.some_method(arg1, arg2, ...) 57 | ``` 58 | is equivalent to 59 | ```python 60 | f = lambda obj: obj.some_method(arg1, arg2, ...) 61 | ``` 62 | ## Composition 63 | #### >> and << 64 | You can use the `>>` operator to *forward* compose expressions 65 | 66 | ```python 67 | f = P + 7 >> math.sqrt #executes left to right 68 | 69 | assert f(2) == 3 # math.sqrt(2 + 7) == math.sqrt(9) == 3 70 | ``` 71 | This is preferred because it is more readable, but you can use the `<<` to compose them *backwards* just like the mathematical definition of function composition 72 | 73 | ```python 74 | f = math.sqrt << P + 7 #executes right to left 75 | 76 | assert f(2) == 3 # math.sqrt(2 + 7) == math.sqrt(9) == 3 77 | ``` 78 | 79 | #### Seq and Pipe 80 | If you need to do a long or complex composition you can use `Seq` (for 'Sequence') instead of many chained `>>` 81 | 82 | ```python 83 | f = Seq( 84 | str, 85 | P + "00", 86 | int, 87 | math.sqrt 88 | ) 89 | 90 | assert f(1) == 10 # sqrt(int("1" + "00")) == sqrt(100) == 10 91 | ``` 92 | If you want to create a composition and directly apply it to an initial value you can use `Pipe` 93 | 94 | ```python 95 | assert 10 == Pipe( 96 | 1, #input 97 | str, # "1" 98 | P + "00", # "1" + "00" == "100" 99 | int, # 100 100 | math.sqrt #sqrt(100) == 10 101 | ) 102 | ``` 103 | 104 | ## Combinators 105 | #### List, Tuple, Set, Dict 106 | There are a couple of combinators like `List`, `Tuple`, `Set`, `Dict` that help you create compound functions that return the container types `list`, `tuple`, `set` and `dict` respectively. For example, you can pass `List` a couple of expressions to get a function that returns a list with the values of these functions 107 | 108 | ```python 109 | f = List( P + 1, P * 10 ) #lambda x: [ x +1, x * 10 ] 110 | 111 | assert f(3) == [ 4, 30 ] # [ 3 + 1, 3 * 10 ] == [ 4, 30 ] 112 | ``` 113 | The same logic applies for `Tuple` and `Set`. With `Dict` you have to use keyword arguments 114 | 115 | ```python 116 | f = Dict( x = P + 1, y = P * 10 ) #lambda x: [ x +1, x * 10 ] 117 | 118 | d = f(3) 119 | 120 | assert d == {{ 'x': 4, 'y': 30 }} # {{ 'x': 3 + 1, 'y': 3 * 10 }} == {{ 'x': 4, 'y': 30 }} 121 | assert d.x == 4 #access d['x'] via field access as d.x 122 | assert d.y == 30 #access d['y'] via field access as d.y 123 | ``` 124 | As you see, `Dict` returns a custom `dict` that also allows *field access*, this is useful because you can use it in combination with `Rec`. 125 | 126 | #### State: Read and Write 127 | Internally all these expressions are implemented in such a way that they not only pass their computed values but also pass a **state** dictionary between them in a functional manner. By reading from and writing to this state dictionary the `Read` and `Write` combinators can help you "save" the state of intermediate computations to read them later 128 | 129 | ```python 130 | assert [70, 30] == Pipe( 131 | 3, 132 | Write(s = P * 10), #s = 3 * 10 == 30 133 | P + 5, #30 + 5 == 35 134 | List( 135 | P * 2 # 35 * 2 == 70 136 | , 137 | Read('s') #s == 30 138 | ) 139 | ) 140 | ``` 141 | If you need to perform many reads inside a list -usually for output- you can use `ReadList` instead 142 | ```python 143 | assert [2, 4, 22] == Pipe( 144 | 1, 145 | Write(a = P + 1), #a = 1 + 1 == 2 146 | Write(b = P * 2), #b = 2 * 2 == 4 147 | P * 5, # 4 * 5 == 20 148 | ReadList('a', 'b', P + 2) # [a, b, 20 + 2] == [2, 4, 22] 149 | ) 150 | ``` 151 | `ReadList` interprets string elements as `Read`s, so the previous is translated to 152 | ```python 153 | List(Read('a'), Read('b'), P + 2) 154 | ``` 155 | 156 | #### Then, Then2, ..., Then5, ThenAt 157 | To create a partial expression from a function e.g. 158 | ```python 159 | def repeat_word(word, times, upper=False): 160 | if upper: 161 | word = word.upper() 162 | 163 | return [ word ] * times 164 | ``` 165 | use the `Then` combinator which accepts a function plus all but the *1st* of its `*args` + `**kwargs` 166 | ```python 167 | f = P[::-1] >> Then(repeat_word, 3) 168 | g = P[::-1] >> Then(repeat_word, 3, upper=True) 169 | 170 | assert f("ward") == ["draw", "draw", "draw"] 171 | assert g("ward") == ["DRAW", "DRAW", "DRAW"] 172 | ``` 173 | and assumes that the *1st* argument of the function will be applied last, e.g. `word` in the case of `repeat_word`. If you need the *2nd* argument to be applied last use `Then2`, and so on. In general you can use `ThenAt(n, f, *args, **kwargs)` where `n` is the position of the argument that will be applied last. Example 174 | ```python 175 | # since map and filter receive the iterable on their second argument, you have to use `Then2` 176 | f = Then2(filter, P % 2 == 0) >> Then2(map, P**2) >> list #lambda x: map(lambda z: z**2, filter(lambda z: z % 2 == 0, x)) 177 | 178 | assert f([1,2,3,4,5]) == [4, 16] #[2**2, 4**2] == [4, 16] 179 | ``` 180 | Be aware that `P` already has the `map` and `filter` methods so you can write the previous more easily as 181 | ```python 182 | f = P.filter(P % 2 == 0) >> P.map(P**2) >> list #lambda x: map(lambda z: z**2, filter(lambda z: z % 2 == 0, x)) 183 | 184 | assert f([1,2,3,4,5]) == [4, 16] #[2**2, 4**2] == [4, 16] 185 | ``` 186 | 187 | #### Val 188 | If you need to create a constant function with a given value use `Val` 189 | ```python 190 | f = Val(42) #lambda x: 42 191 | 192 | assert f("whatever") == 42 193 | ``` 194 | 195 | #### Others 196 | Check out the `With`, `If` and more, combinators on the documentation. The `P` object also offers some useful combinators as methods such as `Not`, `First`, `Last` plus **almost all** python built in functions as methods: 197 | 198 | ```python 199 | f = Obj.split(' ') >> P.map(len) >> sum >> If( (P < 15).Not(), "Great! Got {{0}} letters!".format).Else("Too short, need at-least 15 letters") 200 | 201 | assert f("short frase") == "Too short, need at-least 15 letters" 202 | assert f("some longer frase") == "Great! Got 15 letters!" 203 | ``` 204 | 205 | ## The DSL 206 | Phi has a small omnipresent DSL that has these simple rules: 207 | 208 | 1. Any element of the class `Expression` is an element of the DSL. `P` and all the combinators are of the `Expression` class. 209 | 2. Any callable of arity 1 is an element of the DSL. 210 | 3. The container types `list`, `tuple`, `set`, and `dict` are elements of the DSL. They are translated to their counterparts `List`, `Tuple`, `Set` and `Dict`, their internal elements are forwarded. 211 | 4. Any value `x` that does not comply with any of the previous rules is also an element of the DSL and is translated to `Val(x)`. 212 | 213 | Using the DSL, the expression 214 | 215 | ```python 216 | f = P**2 >> List( P, Val(3), Val(4) ) #lambda x: [ x**2] 217 | 218 | assert f(10) == [ 100, 3, 4 ] # [ 10**2, 3, 4 ] == [ 100, 3, 4 ] 219 | ``` 220 | can be rewritten as 221 | ```python 222 | f = P**2 >> [ P, 3, 4 ] 223 | 224 | assert f(10) == [ 100, 3, 4 ] # [ 10 ** 2, 3, 4 ] == [ 100, 3, 4 ] 225 | ``` 226 | Here the values `3` and `4` are translated to `Val(3)` and `Val(4)` thanks to the *4th* rule, and `[...]` is translated to `List(...)` thanks to the *3rd* rule. Since the DSL is omnipresent you can use it inside any core function, so the previous can be rewritten using `Pipe` as 227 | ```python 228 | assert [ 100, 3, 4 ] == Pipe( 229 | 10, 230 | P**2, # 10**2 == 100 231 | [ P, 3, 4 ] #[ 100, 3, 4 ] 232 | ) 233 | ``` 234 | 235 | #### F 236 | You can *compile* any element to an `Expression` using `F` 237 | ```python 238 | f = F((P + "!!!", 42, Obj.upper())) #Tuple(P + "!!!", Val(42), Obj.upper()) 239 | 240 | assert f("some tuple") == ("some tuple!!!", 42, "SOME TUPLE") 241 | ``` 242 | Other example 243 | ```python 244 | f = F([ P + n for n in range(5) ]) >> [ len, sum ] # lambda x: [ len([ x, x+1, x+2, x+3, x+4]), sum([ x, x+1, x+2, x+3, x+4]) ] 245 | 246 | assert f(10) == [ 5, 60 ] # [ len([10, 11, 12, 13, 14]), sum([10, 11, 12, 13, 14])] == [ 5, (50 + 0 + 1 + 2 + 3 + 4) ] == [ 5, 60 ] 247 | ``` 248 | 249 | ## Fluent Programming 250 | All the functions you've seen are ultimately methods of the `PythonBuilder` class which inherits from the `Expression`, therefore you can also [fluently](https://en.wikipedia.org/wiki/Fluent_interface) chain methods instead of using the `>>` operator. For example 251 | 252 | ```python 253 | f = Dict( 254 | x = 2 * P, 255 | y = P + 1 256 | ).Tuple( 257 | Rec.x + Rec.y, 258 | Rec.y / Rec.x 259 | ) 260 | 261 | assert f(1) == (4, 1) # ( x + y, y / x) == ( 2 + 2, 2 / 2) == ( 4, 1 ) 262 | ``` 263 | This more complicated previous example 264 | ```python 265 | f = Obj.split(' ') >> P.map(len) >> sum >> If( (P < 15).Not(), "Great! Got {{0}} letters!".format).Else("Too short, need at-least 15 letters") 266 | 267 | assert f("short frase") == "Too short, need at-least 15 letters" 268 | assert f("some longer frase") == "Great! Got 15 letters!" 269 | ``` 270 | can be be rewritten as 271 | ```python 272 | f = ( 273 | Obj.split(' ') 274 | .map(len) 275 | .sum() 276 | .If( (P < 15).Not(), 277 | "Great! Got {{0}} letters!".format 278 | ).Else( 279 | "Too short, need at-least 15 letters" 280 | ) 281 | ) 282 | 283 | assert f("short frase") == "Too short, need at-least 15 letters" 284 | assert f("some longer frase") == "Great! Got 15 letters!" 285 | ``` 286 | 287 | ## Integrability 288 | #### Register, Register2, ..., Register5, RegistarAt 289 | If you want to have custom expressions to deal with certain data types, you can create a custom class that inherits from `Builder` or `PythonBuilder` 290 | ```python 291 | from phi import PythonBuilder 292 | 293 | class MyBuilder(PythonBuilder): 294 | pass 295 | 296 | M = MyBuilder() 297 | ``` 298 | and register your function in it using the `Register` class method 299 | 300 | ```python 301 | def remove_longer_than(some_list, n): 302 | return [ elem from elem in some_list if len(elem) <= n ] 303 | 304 | MyBuilder.Register(remove_longer_than, "my.lib.") 305 | ``` 306 | Or better even use `Register` as a decorator 307 | ```python 308 | @MyBuilder.Register("my.lib.") 309 | def remove_longer_than(some_list, n): 310 | return [ elem for elem in some_list if len(elem) <= n ] 311 | ``` 312 | 313 | Now the method `MyBuilder.remove_longer_than` exists on this class. You can then use it like this 314 | ```python 315 | f = Obj.lower() >> Obj.split(' ') >> M.remove_longer_than(6) 316 | 317 | assert f("SoMe aRe LONGGGGGGGGG") == ["some", "are"] 318 | ``` 319 | As you see the argument `n = 6` was partially applied to `remove_longer_than`, an expression which waits for the `some_list` argument to be returned. Internally the `Registar*` method family uses the `Then*` method family. 320 | 321 | #### PatchAt 322 | If you want to register a batch of functions from a module or class automatically you can use the `PatchAt` class method. It's an easy way to integrate an entire module to Phi's DSL. See `PatchAt`. 323 | 324 | #### Libraries 325 | Phi currently powers the following libraries that integrate with its DSL: 326 | 327 | * [PythonBuilder](https://cgarciae.github.io/phi/python_builder.m.html) : helps you integrate Python's built-in functions and keywords into the phi DSL and it also includes a bunch of useful helpers for common stuff. `phi`'s global `P` object is an instance of this class. [Shipped with Phi] 328 | * [TensorBuilder](https://github.com/cgarciae/tensorbuilder): a TensorFlow library enables you to easily create complex deep neural networks by leveraging the phi DSL to help define their structure. 329 | * NumpyBuilder: Comming soon! 330 | 331 | ## Documentation 332 | Check out the [complete documentation](https://cgarciae.github.io/phi/). 333 | 334 | ## More Examples 335 | The global `phi.P` object exposes most of the API and preferably should be imported directly. The most simple thing the DSL does is function composition: 336 | 337 | ```python 338 | from phi.api import * 339 | 340 | def add1(x): return x + 1 341 | def mul3(x): return x * 3 342 | 343 | x = Pipe( 344 | 1.0, #input 1 345 | add1, #1 + 1 == 2 346 | mul3 #2 * 3 == 6 347 | ) 348 | 349 | assert x == 6 350 | ``` 351 | 352 | Use phi [lambdas](https://cgarciae.github.io/phi/lambdas.m.html) to create the functions 353 | 354 | ```python 355 | from phi.api import * 356 | 357 | x = Pipe( 358 | 1.0, #input 1 359 | P + 1, #1 + 1 == 2 360 | P * 3 #2 * 3 == 6 361 | ) 362 | 363 | assert x == 6 364 | ``` 365 | 366 | Create a branched computation instead 367 | 368 | ```python 369 | from phi.api import * 370 | 371 | [x, y] = Pipe( 372 | 1.0, #input 1 373 | [ 374 | P + 1 #1 + 1 == 2 375 | , 376 | P * 3 #1 * 3 == 3 377 | ] 378 | ) 379 | 380 | assert x == 2 381 | assert y == 3 382 | ``` 383 | 384 | Compose it with a function equivalent to `f(x) = (x + 3) / (x + 1)` 385 | 386 | ```python 387 | from phi.api import * 388 | 389 | [x, y] = Pipe( 390 | 1.0, #input 1 391 | (P + 3) / (P + 1), #(1 + 3) / (1 + 1) == 4 / 2 == 2 392 | [ 393 | P + 1 #2 + 1 == 3 394 | , 395 | P * 3 #2 * 3 == 6 396 | ] 397 | ) 398 | 399 | assert x == 3 400 | assert y == 6 401 | ``` 402 | 403 | Give names to the branches 404 | 405 | ```python 406 | from phi.api import * 407 | 408 | result = Pipe( 409 | 1.0, #input 1 410 | (P + 3) / (P + 1), #(1 + 3) / (1 + 1) == 4 / 2 == 2 411 | dict( 412 | x = P + 1 #2 + 1 == 3 413 | , 414 | y = P * 3 #2 * 3 == 6 415 | ) 416 | ) 417 | 418 | assert result.x == 3 419 | assert result.y == 6 420 | ``` 421 | 422 | Divide `x` by `y`. 423 | 424 | ```python 425 | from phi.api import * 426 | 427 | result = Pipe( 428 | 1.0, #input 1 429 | (P + 3) / (P + 1), #(1 + 3) / (1 + 1) == 4 / 2 == 2 430 | dict( 431 | x = P + 1 #2 + 1 == 3 432 | , 433 | y = P * 3 #2 * 3 == 6 434 | ), 435 | Rec.x / Rec.y #3 / 6 == 0.5 436 | ) 437 | 438 | assert result == 0.5 439 | ``` 440 | 441 | Save the value from the `(P + 3) / (P + 1)` computation as `s` and load it at the end in a branch 442 | 443 | ```python 444 | from phi.api import * 445 | 446 | [result, s] = Pipe( 447 | 1.0, #input 1 448 | Write(s = (P + 3) / (P + 1)), #s = 4 / 2 == 2 449 | dict( 450 | x = P + 1 #2 + 1 == 3 451 | , 452 | y = P * 3 #2 * 3 == 6 453 | ), 454 | [ 455 | Rec.x / Rec.y #3 / 6 == 0.5 456 | , 457 | Read('s') #s == 2 458 | ] 459 | ) 460 | 461 | assert result == 0.5 462 | assert s == 2 463 | ``` 464 | 465 | Add 3 to the loaded `s` for fun and profit 466 | 467 | ```python 468 | from phi.api import * 469 | 470 | [result, s] = Pipe( 471 | 1.0, #input 1 472 | Write(s = (P + 3) / (P + 1)), #s = 4 / 2 == 2 473 | dict( 474 | x = P + 1 #2 + 1 == 3 475 | , 476 | y = P * 3 #2 * 3 == 6 477 | ), 478 | [ 479 | Rec.x / Rec.y #3 / 6 == 0.5 480 | , 481 | Read('s') + 3 # 2 + 3 == 5 482 | ] 483 | ) 484 | 485 | assert result == 0.5 486 | assert s == 5 487 | ``` 488 | 489 | Use the `Read` and `Write` field access lambda style just because 490 | 491 | ```python 492 | from phi.api import * 493 | 494 | [result, s] = Pipe( 495 | 1.0, #input 1 496 | (P + 3) / (P + 1), #4 / 2 == 2 497 | Write.s, #s = 2 498 | dict( 499 | x = P + 1 #2 + 1 == 3 500 | , 501 | y = P * 3 #2 * 3 == 6 502 | ), 503 | [ 504 | Rec.x / Rec.y #3 / 6 == 0.5 505 | , 506 | Read.s + 3 # 2 + 3 == 5 507 | ] 508 | ) 509 | 510 | assert result == 0.5 511 | assert s == 5 512 | ``` 513 | 514 | Add an input `Val` of 9 on a branch and add to it 1 just for the sake of it 515 | 516 | ```python 517 | from phi.api import * 518 | 519 | [result, s, val] = Pipe( 520 | 1.0, #input 1 521 | (P + 3) / (P + 1), Write.s, #4 / 2 == 2, saved as 's' 522 | dict( 523 | x = P + 1 #2 + 1 == 3 524 | , 525 | y = P * 3 #2 * 3 == 6 526 | ), 527 | [ 528 | Rec.x / Rec.y #3 / 6 == 0.5 529 | , 530 | Read.s + 3 # 2 + 3 == 5 531 | , 532 | Val(9) + 1 #input 9 and add 1, gives 10 533 | ] 534 | ) 535 | 536 | assert result == 0.5 537 | assert s == 5 538 | assert val == 10 539 | ``` 540 | 541 | Do the previous only if `y > 7` else return `"Sorry, come back latter."` 542 | 543 | ```python 544 | from phi.api import * 545 | 546 | [result, s, val] = Pipe( 547 | 1.0, #input 1 548 | (P + 3) / (P + 1), Write.s, #4 / 2 == 2, saved as 's' 549 | dict( 550 | x = P + 1 #2 + 1 == 3 551 | , 552 | y = P * 3 #2 * 3 == 6 553 | ), 554 | [ 555 | Rec.x / Rec.y #3 / 6 == 0.5 556 | , 557 | Read.s + 3 # 2 + 3 == 5 558 | , 559 | If( Rec.y > 7, 560 | Val(9) + 1 #input 9 and add 1, gives 10 561 | ).Else( 562 | "Sorry, come back latter." 563 | ) 564 | ] 565 | ) 566 | 567 | assert result == 0.5 568 | assert s == 5 569 | assert val == "Sorry, come back latter." 570 | ``` 571 | 572 | Now, what you have to understand that everything you've done with these expression is to create and apply a single function. Using `Seq` we can get the standalone function and then use it to get the same values as before 573 | 574 | ```python 575 | from phi.api import * 576 | 577 | f = Seq( 578 | (P + 3) / (P + 1), Write.s, #4 / 2 == 2, saved as 's' 579 | dict( 580 | x = P + 1 #2 + 1 == 3 581 | , 582 | y = P * 3 #2 * 3 == 6 583 | ), 584 | [ 585 | Rec.x / Rec.y #3 / 6 == 0.5 586 | , 587 | Read.s + 3 # 2 + 3 == 5 588 | , 589 | If( Rec.y > 7, 590 | Val(9) + 1 #input 9 and add 1, gives 10 591 | ).Else( 592 | "Sorry, come back latter." 593 | ) 594 | ] 595 | ) 596 | 597 | [result, s, val] = f(1.0) 598 | 599 | assert result == 0.5 600 | assert s == 5 601 | assert val == "Sorry, come back latter." 602 | ``` 603 | ### Even More Examples 604 | 605 | ```python 606 | from phi.api import * 607 | 608 | avg_word_length = Pipe( 609 | "1 22 333", 610 | Obj.split(" "), # ['1', '22', '333'] 611 | P.map(len), # [1, 2, 3] 612 | P.sum() / P.len() # sum([1,2,3]) / len([1,2,3]) == 6 / 3 == 2 613 | ) 614 | 615 | assert 2 == avg_word_length 616 | ``` 617 | 618 | ```python 619 | from phi.api import * 620 | 621 | assert False == Pipe( 622 | [1,2,3,4], P 623 | .filter(P % 2 != 0) #[1, 3], keeps odds 624 | .Contains(4) #4 in [1, 3] == False 625 | ) 626 | ``` 627 | 628 | ```python 629 | from phi.api import * 630 | 631 | assert {{'a': 97, 'b': 98, 'c': 99}} == Pipe( 632 | "a b c", Obj 633 | .split(' ').Write.keys # keys = ['a', 'b', 'c'] 634 | .map(ord), # [ord('a'), ord('b'), ord('c')] == [97, 98, 99] 635 | lambda it: zip(Ref.keys, it), # [('a', 97), ('b', 98), ('c', 99)] 636 | dict # {{'a': 97, 'b': 98, 'c': 99}} 637 | ) 638 | ``` 639 | 640 | ## Installation 641 | 642 | pip install phi 643 | 644 | 645 | #### Bleeding Edge 646 | 647 | pip install git+https://github.com/cgarciae/phi.git@develop 648 | 649 | ## Status 650 | * Version: **{0}**. 651 | * Documentation coverage: 100%. Please create an issue if documentation is unclear, it is a high priority of this library. 652 | * Milestone: reach 1.0.0 after feedback from the community. 653 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Phi 2 | Phi library for functional programming in Python that intends to remove as much of the pain as possible from your functional programming experience in Python. 3 | 4 | ## Import 5 | For demonstration purposes we will import right now everything we will need for the rest of the exercises like this 6 | 7 | ```python 8 | from phi.api import * 9 | ``` 10 | but you can also import just what you need from the `phi` module. 11 | 12 | ## Math-like Lambdas 13 | 14 | #### Operators 15 | 16 | Using the `P` object you can create quick lambdas using any operator. You can write things like 17 | 18 | ```python 19 | f = (P * 6) / (P + 2) # lambda x: (x * 6) / (x + 2) 20 | 21 | assert f(2) == 3 # (2 * 6) / (2 + 2) == 12 / 4 == 3 22 | ``` 23 | 24 | where the expression for `f` is equivalent to 25 | 26 | ```python 27 | f = lambda x: (x * 6) / (x + 2) 28 | ``` 29 | 30 | #### getitem 31 | You can also use the `P` object to create lambdas that access the items of a collection 32 | 33 | ```python 34 | f = P[0] + P[-1] # lambda x: x[0] + x[-1] 35 | 36 | assert f([1,2,3,4]) == 5 #1 + 4 == 5 37 | ``` 38 | 39 | #### field access 40 | If you want create lambdas that access the field of some entity you can use the `Rec` (for Record) object an call that field on it 41 | ```python 42 | from collections import namedtuple 43 | Point = namedtuple('Point', ['x', 'y']) 44 | 45 | f = Rec.x + Rec.y # lambda p: p.x + p.y 46 | 47 | assert f(Point(3, 4)) == 7 # point.x + point.y == 3 + 4 == 7 48 | ``` 49 | #### method calling 50 | If you want to create a lambda that calls the method of an object you use the `Obj` object and call that method on it with the parameters 51 | 52 | ```python 53 | f = Obj.upper() + ", " + Obj.lower() # lambda s: s.upper() + ", " + s.lower() 54 | 55 | assert f("HEllo") == "HELLO, hello" # "HEllo".upper() + ", " + "HEllo".lower() == "HELLO" + ", " + "hello" == "HELLO, hello" 56 | ``` 57 | 58 | Here no parameters were needed but in general 59 | 60 | ```python 61 | f = Obj.some_method(arg1, arg2, ...) #lambda obj: obj.some_method(arg1, arg2, ...) 62 | ``` 63 | 64 | is equivalent to 65 | 66 | ```python 67 | f = lambda obj: obj.some_method(arg1, arg2, ...) 68 | ``` 69 | 70 | ## Composition 71 | #### >> and << 72 | You can use the `>>` operator to *forward* compose expressions 73 | 74 | ```python 75 | f = P + 7 >> math.sqrt #executes left to right 76 | 77 | assert f(2) == 3 # math.sqrt(2 + 7) == math.sqrt(9) == 3 78 | ``` 79 | 80 | This is preferred because it is more readable, but you can use the `<<` to compose them *backwards* just like the mathematical definition of function composition 81 | 82 | ```python 83 | f = math.sqrt << P + 7 #executes right to left 84 | 85 | assert f(2) == 3 # math.sqrt(2 + 7) == math.sqrt(9) == 3 86 | ``` 87 | 88 | #### Seq and Pipe 89 | If you need to do a long or complex composition you can use `Seq` (for 'Sequence') instead of many chained `>>` 90 | 91 | ```python 92 | f = Seq( 93 | str, 94 | P + "00", 95 | int, 96 | math.sqrt 97 | ) 98 | 99 | assert f(1) == 10 # sqrt(int("1" + "00")) == sqrt(100) == 10 100 | ``` 101 | 102 | If you want to create a composition and directly apply it to an initial value you can use `Pipe` 103 | 104 | ```python 105 | assert 10 == Pipe( 106 | 1, #input 107 | str, # "1" 108 | P + "00", # "1" + "00" == "100" 109 | int, # 100 110 | math.sqrt #sqrt(100) == 10 111 | ) 112 | ``` 113 | 114 | ## Combinators 115 | #### List, Tuple, Set, Dict 116 | There are a couple of combinators like `List`, `Tuple`, `Set`, `Dict` that help you create compound functions that return the container types `list`, `tuple`, `set` and `dict` respectively. For example, you can pass `List` a couple of expressions to get a function that returns a list with the values of these functions 117 | 118 | ```python 119 | f = List( P + 1, P * 10 ) #lambda x: [ x +1, x * 10 ] 120 | 121 | assert f(3) == [ 4, 30 ] # [ 3 + 1, 3 * 10 ] == [ 4, 30 ] 122 | ``` 123 | 124 | The same logic applies for `Tuple` and `Set`. With `Dict` you have to use keyword arguments 125 | 126 | ```python 127 | f = Dict( x = P + 1, y = P * 10 ) #lambda x: [ x +1, x * 10 ] 128 | 129 | d = f(3) 130 | 131 | assert d == { 'x': 4, 'y': 30 } # { 'x': 3 + 1, 'y': 3 * 10 } == { 'x': 4, 'y': 30 } 132 | assert d.x == 4 #access d['x'] via field access as d.x 133 | assert d.y == 30 #access d['y'] via field access as d.y 134 | ``` 135 | 136 | As you see, `Dict` returns a custom `dict` that also allows *field access*, this is useful because you can use it in combination with `Rec`. 137 | 138 | #### State: Read and Write 139 | Internally all these expressions are implemented in such a way that they not only pass their computed values but also pass a **state** dictionary between them in a functional manner. By reading from and writing to this state dictionary the `Read` and `Write` combinators can help you "save" the state of intermediate computations to read them later 140 | 141 | ```python 142 | assert [70, 30] == Pipe( 143 | 3, 144 | Write(s = P * 10), #s = 3 * 10 == 30 145 | P + 5, #30 + 5 == 35 146 | List( 147 | P * 2 # 35 * 2 == 70 148 | , 149 | Read('s') #s == 30 150 | ) 151 | ) 152 | ``` 153 | 154 | If you need to perform many reads inside a list -usually for output- you can use `ReadList` instead 155 | 156 | ```python 157 | assert [2, 4, 22] == Pipe( 158 | 1, 159 | Write(a = P + 1), #a = 1 + 1 == 2 160 | Write(b = P * 2), #b = 2 * 2 == 4 161 | P * 5, # 4 * 5 == 20 162 | ReadList('a', 'b', P + 2) # [a, b, 20 + 2] == [2, 4, 22] 163 | ) 164 | ``` 165 | `ReadList` interprets string elements as `Read`s, so the previous is translated to 166 | ```python 167 | List(Read('a'), Read('b'), P + 2) 168 | ``` 169 | 170 | #### Then, Then2, ..., Then5, ThenAt 171 | To create a partial expression from a function e.g. 172 | 173 | ```python 174 | def repeat_word(word, times, upper=False): 175 | if upper: 176 | word = word.upper() 177 | 178 | return [ word ] * times 179 | ``` 180 | 181 | use the `Then` combinator which accepts a function plus all but the *1st* of its `*args` + `**kwargs` 182 | 183 | ```python 184 | f = P[::-1] >> Then(repeat_word, 3) 185 | g = P[::-1] >> Then(repeat_word, 3, upper=True) 186 | 187 | assert f("ward") == ["draw", "draw", "draw"] 188 | assert g("ward") == ["DRAW", "DRAW", "DRAW"] 189 | ``` 190 | 191 | and assumes that the *1st* argument of the function will be applied last, e.g. `word` in the case of `repeat_word`. If you need the *2nd* argument to be applied last use `Then2`, and so on. In general you can use `ThenAt(n, f, *args, **kwargs)` where `n` is the position of the argument that will be applied last. For example 192 | 193 | ```python 194 | # since map and filter receive the iterable on their second argument, you have to use `Then2` 195 | f = Then2(filter, P % 2 == 0) >> Then2(map, P**2) >> list #lambda x: map(lambda z: z**2, filter(lambda z: z % 2 == 0, x)) 196 | 197 | assert f([1,2,3,4,5]) == [4, 16] #[2**2, 4**2] == [4, 16] 198 | ``` 199 | 200 | Be aware that `P` already has the `map` and `filter` methods so you can write the previous more easily as 201 | 202 | ```python 203 | f = P.filter(P % 2 == 0) >> P.map(P**2) >> list #lambda x: map(lambda z: z**2, filter(lambda z: z % 2 == 0, x)) 204 | 205 | assert f([1,2,3,4,5]) == [4, 16] #[2**2, 4**2] == [4, 16] 206 | ``` 207 | 208 | #### Val 209 | If you need to create a constant function with a given value use `Val` 210 | 211 | ```python 212 | f = Val(42) #lambda x: 42 213 | 214 | assert f("whatever") == 42 215 | ``` 216 | 217 | #### Others 218 | Check out the `With`, `If` and more, combinators on the documentation. The `P` object also offers some useful combinators as methods such as `Not`, `First`, `Last` plus **almost all** python built in functions as methods: 219 | 220 | ```python 221 | f = Obj.split(' ') >> P.map(len) >> sum >> If( (P < 15).Not(), "Great! Got {0} letters!".format).Else("Too short, need at-least 15 letters") 222 | 223 | assert f("short frase") == "Too short, need at-least 15 letters" 224 | assert f("some longer frase") == "Great! Got 15 letters!" 225 | ``` 226 | 227 | ## The DSL 228 | Phi has a small omnipresent DSL that has these simple rules: 229 | 230 | 1. Any element of the class `Expression` is an element of the DSL. `P` and all the combinators are of the `Expression` class. 231 | 2. Any callable of arity 1 is an element of the DSL. 232 | 3. The container types `list`, `tuple`, `set`, and `dict` are elements of the DSL. They are translated to their counterparts `List`, `Tuple`, `Set` and `Dict`, their internal elements are forwarded. 233 | 4. Any value `x` that does not comply with any of the previous rules is also an element of the DSL and is translated to `Val(x)`. 234 | 235 | Using the DSL, the expression 236 | 237 | ```python 238 | f = P**2 >> List( P, Val(3), Val(4) ) #lambda x: [ x**2] 239 | 240 | assert f(10) == [ 100, 3, 4 ] # [ 10**2, 3, 4 ] == [ 100, 3, 4 ] 241 | ``` 242 | 243 | can be rewritten as 244 | 245 | ```python 246 | f = P**2 >> [ P, 3, 4 ] 247 | 248 | assert f(10) == [ 100, 3, 4 ] # [ 10 ** 2, 3, 4 ] == [ 100, 3, 4 ] 249 | ``` 250 | 251 | Here the values `3` and `4` are translated to `Val(3)` and `Val(4)` thanks to the *4th* rule, and `[...]` is translated to `List(...)` thanks to the *3rd* rule. Since the DSL is omnipresent you can use it inside any core function, so the previous can be rewritten using `Pipe` as 252 | 253 | ```python 254 | assert [ 100, 3, 4 ] == Pipe( 255 | 10, 256 | P**2, # 10**2 == 100 257 | [ P, 3, 4 ] #[ 100, 3, 4 ] 258 | ) 259 | ``` 260 | 261 | #### F 262 | You can *compile* any element to an `Expression` using `F` 263 | 264 | ```python 265 | f = F((P + "!!!", 42, Obj.upper())) #Tuple(P + "!!!", Val(42), Obj.upper()) 266 | 267 | assert f("some tuple") == ("some tuple!!!", 42, "SOME TUPLE") 268 | ``` 269 | 270 | Other example 271 | 272 | ```python 273 | f = F([ P + n for n in range(5) ]) >> [ len, sum ] # lambda x: [ len([ x, x+1, x+2, x+3, x+4]), sum([ x, x+1, x+2, x+3, x+4]) ] 274 | 275 | assert f(10) == [ 5, 60 ] # [ len([10, 11, 12, 13, 14]), sum([10, 11, 12, 13, 14])] == [ 5, (50 + 0 + 1 + 2 + 3 + 4) ] == [ 5, 60 ] 276 | ``` 277 | 278 | ## Fluent Programming 279 | All the functions you've seen are ultimately methods of the `PythonBuilder` class which inherits from the `Expression`, therefore you can also [fluently](https://en.wikipedia.org/wiki/Fluent_interface) chain methods instead of using the `>>` operator. For example 280 | 281 | ```python 282 | f = Dict( 283 | x = 2 * P, 284 | y = P + 1 285 | ).Tuple( 286 | Rec.x + Rec.y, 287 | Rec.y / Rec.x 288 | ) 289 | 290 | assert f(1) == (4, 1) # ( x + y, y / x) == ( 2 + 2, 2 / 2) == ( 4, 1 ) 291 | ``` 292 | 293 | This more complicated previous example 294 | 295 | ```python 296 | f = Obj.split(' ') >> P.map(len) >> sum >> If( (P < 15).Not(), "Great! Got {0} letters!".format).Else("Too short, need at-least 15 letters") 297 | 298 | assert f("short frase") == "Too short, need at-least 15 letters" 299 | assert f("some longer frase") == "Great! Got 15 letters!" 300 | ``` 301 | 302 | can be be rewritten as 303 | 304 | ```python 305 | f = ( 306 | Obj.split(' ') 307 | .map(len) 308 | .sum() 309 | .If( (P < 15).Not(), 310 | "Great! Got {0} letters!".format 311 | ).Else( 312 | "Too short, need at-least 15 letters" 313 | ) 314 | ) 315 | 316 | assert f("short frase") == "Too short, need at-least 15 letters" 317 | assert f("some longer frase") == "Great! Got 15 letters!" 318 | ``` 319 | 320 | ## Integrability 321 | #### Register, Register2, ..., Register5, RegistarAt 322 | If you want to have custom expressions to deal with certain data types, you can create a custom class that inherits from `Builder` or `PythonBuilder` 323 | 324 | ```python 325 | from phi import PythonBuilder 326 | 327 | class MyBuilder(PythonBuilder): 328 | pass 329 | 330 | M = MyBuilder() 331 | ``` 332 | 333 | and register your function in it using the `Register` class method 334 | 335 | ```python 336 | def remove_longer_than(some_list, n): 337 | return [ elem from elem in some_list if len(elem) <= n ] 338 | 339 | MyBuilder.Register(remove_longer_than, "my.lib.") 340 | ``` 341 | Or better even use `Register` as a decorator 342 | ```python 343 | @MyBuilder.Register("my.lib.") 344 | def remove_longer_than(some_list, n): 345 | return [ elem for elem in some_list if len(elem) <= n ] 346 | ``` 347 | 348 | Now the method `MyBuilder.remove_longer_than` exists on this class. You can then use it like this 349 | 350 | ```python 351 | f = Obj.lower() >> Obj.split(' ') >> M.remove_longer_than(6) 352 | 353 | assert f("SoMe aRe LONGGGGGGGGG") == ["some", "are"] 354 | ``` 355 | 356 | As you see the argument `n = 6` was partially applied to `remove_longer_than`, an expression which waits for the `some_list` argument to be returned. Internally the `Registar*` method family uses the `Then*` method family. 357 | 358 | #### PatchAt 359 | If you want to register a batch of functions from a module or class automatically you can use the `PatchAt` class method. It's an easy way to integrate an entire module to Phi's DSL. See `PatchAt`. 360 | 361 | #### Libraries 362 | Phi currently powers the following libraries that integrate with its DSL: 363 | 364 | * [PythonBuilder](https://cgarciae.github.io/phi/python_builder.m.html) : helps you integrate Python's built-in functions and keywords into the phi DSL and it also includes a bunch of useful helpers for common stuff. `phi`'s global `P` object is an instance of this class. [Shipped with Phi] 365 | * [TensorBuilder](https://github.com/cgarciae/tensorbuilder): a TensorFlow library enables you to easily create complex deep neural networks by leveraging the phi DSL to help define their structure. 366 | * NumpyBuilder: Comming soon! 367 | 368 | ## Documentation 369 | Check out the [complete documentation](https://cgarciae.github.io/phi/). 370 | 371 | ## More Examples 372 | The global `phi.P` object exposes most of the API and preferably should be imported directly. The most simple thing the DSL does is function composition: 373 | 374 | ```python 375 | from phi.api import * 376 | 377 | def add1(x): return x + 1 378 | def mul3(x): return x * 3 379 | 380 | x = Pipe( 381 | 1.0, #input 1 382 | add1, #1 + 1 == 2 383 | mul3 #2 * 3 == 6 384 | ) 385 | 386 | assert x == 6 387 | ``` 388 | 389 | Use phi [lambdas](https://cgarciae.github.io/phi/lambdas.m.html) to create the functions 390 | 391 | ```python 392 | from phi.api import * 393 | 394 | x = Pipe( 395 | 1.0, #input 1 396 | P + 1, #1 + 1 == 2 397 | P * 3 #2 * 3 == 6 398 | ) 399 | 400 | assert x == 6 401 | ``` 402 | 403 | Create a branched computation instead 404 | 405 | ```python 406 | from phi.api import * 407 | 408 | [x, y] = Pipe( 409 | 1.0, #input 1 410 | [ 411 | P + 1 #1 + 1 == 2 412 | , 413 | P * 3 #1 * 3 == 3 414 | ] 415 | ) 416 | 417 | assert x == 2 418 | assert y == 3 419 | ``` 420 | 421 | Compose it with a function equivalent to `f(x) = (x + 3) / (x + 1)` 422 | 423 | ```python 424 | from phi.api import * 425 | 426 | [x, y] = Pipe( 427 | 1.0, #input 1 428 | (P + 3) / (P + 1), #(1 + 3) / (1 + 1) == 4 / 2 == 2 429 | [ 430 | P + 1 #2 + 1 == 3 431 | , 432 | P * 3 #2 * 3 == 6 433 | ] 434 | ) 435 | 436 | assert x == 3 437 | assert y == 6 438 | ``` 439 | 440 | Give names to the branches 441 | 442 | ```python 443 | from phi.api import * 444 | 445 | result = Pipe( 446 | 1.0, #input 1 447 | (P + 3) / (P + 1), #(1 + 3) / (1 + 1) == 4 / 2 == 2 448 | dict( 449 | x = P + 1 #2 + 1 == 3 450 | , 451 | y = P * 3 #2 * 3 == 6 452 | ) 453 | ) 454 | 455 | assert result.x == 3 456 | assert result.y == 6 457 | ``` 458 | 459 | Divide `x` by `y`. 460 | 461 | ```python 462 | from phi.api import * 463 | 464 | result = Pipe( 465 | 1.0, #input 1 466 | (P + 3) / (P + 1), #(1 + 3) / (1 + 1) == 4 / 2 == 2 467 | dict( 468 | x = P + 1 #2 + 1 == 3 469 | , 470 | y = P * 3 #2 * 3 == 6 471 | ), 472 | Rec.x / Rec.y #3 / 6 == 0.5 473 | ) 474 | 475 | assert result == 0.5 476 | ``` 477 | 478 | Save the value from the `(P + 3) / (P + 1)` computation as `s` and load it at the end in a branch 479 | 480 | ```python 481 | from phi.api import * 482 | 483 | [result, s] = Pipe( 484 | 1.0, #input 1 485 | Write(s = (P + 3) / (P + 1)), #s = 4 / 2 == 2 486 | dict( 487 | x = P + 1 #2 + 1 == 3 488 | , 489 | y = P * 3 #2 * 3 == 6 490 | ), 491 | [ 492 | Rec.x / Rec.y #3 / 6 == 0.5 493 | , 494 | Read('s') #s == 2 495 | ] 496 | ) 497 | 498 | assert result == 0.5 499 | assert s == 2 500 | ``` 501 | 502 | Add 3 to the loaded `s` for fun and profit 503 | 504 | ```python 505 | from phi.api import * 506 | 507 | [result, s] = Pipe( 508 | 1.0, #input 1 509 | Write(s = (P + 3) / (P + 1)), #s = 4 / 2 == 2 510 | dict( 511 | x = P + 1 #2 + 1 == 3 512 | , 513 | y = P * 3 #2 * 3 == 6 514 | ), 515 | [ 516 | Rec.x / Rec.y #3 / 6 == 0.5 517 | , 518 | Read('s') + 3 # 2 + 3 == 5 519 | ] 520 | ) 521 | 522 | assert result == 0.5 523 | assert s == 5 524 | ``` 525 | 526 | Use the `Read` and `Write` field access lambda style just because 527 | 528 | ```python 529 | from phi.api import * 530 | 531 | [result, s] = Pipe( 532 | 1.0, #input 1 533 | (P + 3) / (P + 1), #4 / 2 == 2 534 | Write.s, #s = 2 535 | dict( 536 | x = P + 1 #2 + 1 == 3 537 | , 538 | y = P * 3 #2 * 3 == 6 539 | ), 540 | [ 541 | Rec.x / Rec.y #3 / 6 == 0.5 542 | , 543 | Read.s + 3 # 2 + 3 == 5 544 | ] 545 | ) 546 | 547 | assert result == 0.5 548 | assert s == 5 549 | ``` 550 | 551 | Add an input `Val` of 9 on a branch and add to it 1 just for the sake of it 552 | 553 | ```python 554 | from phi.api import * 555 | 556 | [result, s, val] = Pipe( 557 | 1.0, #input 1 558 | (P + 3) / (P + 1), Write.s, #4 / 2 == 2, saved as 's' 559 | dict( 560 | x = P + 1 #2 + 1 == 3 561 | , 562 | y = P * 3 #2 * 3 == 6 563 | ), 564 | [ 565 | Rec.x / Rec.y #3 / 6 == 0.5 566 | , 567 | Read.s + 3 # 2 + 3 == 5 568 | , 569 | Val(9) + 1 #input 9 and add 1, gives 10 570 | ] 571 | ) 572 | 573 | assert result == 0.5 574 | assert s == 5 575 | assert val == 10 576 | ``` 577 | 578 | Do the previous only if `y > 7` else return `"Sorry, come back later."` 579 | 580 | ```python 581 | from phi.api import * 582 | 583 | [result, s, val] = Pipe( 584 | 1.0, #input 1 585 | (P + 3) / (P + 1), Write.s, #4 / 2 == 2, saved as 's' 586 | dict( 587 | x = P + 1 #2 + 1 == 3 588 | , 589 | y = P * 3 #2 * 3 == 6 590 | ), 591 | [ 592 | Rec.x / Rec.y #3 / 6 == 0.5 593 | , 594 | Read.s + 3 # 2 + 3 == 5 595 | , 596 | If( Rec.y > 7, 597 | Val(9) + 1 #input 9 and add 1, gives 10 598 | ).Else( 599 | "Sorry, come back later." 600 | ) 601 | ] 602 | ) 603 | 604 | assert result == 0.5 605 | assert s == 5 606 | assert val == "Sorry, come back later." 607 | ``` 608 | 609 | Now, what you have to understand that everything you've done with these expression is to create and apply a single function. Using `Seq` we can get the standalone function and then use it to get the same values as before 610 | 611 | ```python 612 | from phi.api import * 613 | 614 | f = Seq( 615 | (P + 3) / (P + 1), Write.s, #4 / 2 == 2, saved as 's' 616 | dict( 617 | x = P + 1 #2 + 1 == 3 618 | , 619 | y = P * 3 #2 * 3 == 6 620 | ), 621 | [ 622 | Rec.x / Rec.y #3 / 6 == 0.5 623 | , 624 | Read.s + 3 # 2 + 3 == 5 625 | , 626 | If( Rec.y > 7, 627 | Val(9) + 1 #input 9 and add 1, gives 10 628 | ).Else( 629 | "Sorry, come back later." 630 | ) 631 | ] 632 | ) 633 | 634 | [result, s, val] = f(1.0) 635 | 636 | assert result == 0.5 637 | assert s == 5 638 | assert val == "Sorry, come back later." 639 | ``` 640 | 641 | ### Even More Examples 642 | 643 | ```python 644 | from phi.api import * 645 | 646 | avg_word_length = Pipe( 647 | "1 22 333", 648 | Obj.split(" "), # ['1', '22', '333'] 649 | P.map(len), # [1, 2, 3] 650 | P.sum() / P.len() # sum([1,2,3]) / len([1,2,3]) == 6 / 3 == 2 651 | ) 652 | 653 | assert 2 == avg_word_length 654 | ``` 655 | 656 | ```python 657 | from phi.api import * 658 | 659 | assert False == Pipe( 660 | [1,2,3,4], P 661 | .filter(P % 2 != 0) #[1, 3], keeps odds 662 | .Contains(4) #4 in [1, 3] == False 663 | ) 664 | ``` 665 | 666 | ```python 667 | from phi.api import * 668 | 669 | assert {'a': 97, 'b': 98, 'c': 99} == Pipe( 670 | "a b c", Obj 671 | .split(' ').Write.keys # keys = ['a', 'b', 'c'] 672 | .map(ord), # [ord('a'), ord('b'), ord('c')] == [97, 98, 99] 673 | lambda it: zip(Ref.keys, it), # [('a', 97), ('b', 98), ('c', 99)] 674 | dict # {'a': 97, 'b': 98, 'c': 99} 675 | ) 676 | ``` 677 | 678 | ## Installation 679 | 680 | ```sh 681 | pip install phi 682 | ``` 683 | 684 | #### Bleeding Edge Release 685 | 686 | ```sh 687 | pip install git+https://github.com/cgarciae/phi.git@develop 688 | ``` 689 | 690 | ## Status 691 | * Version: ![Latest Tagged Release Number](https://img.shields.io/github/tag/cgarciae/phi.svg?%3FlongCache=true&label=). 692 | * Documentation coverage: 100%. Please create an issue if documentation is unclear, it is a high priority of this library. 693 | * Milestone: reach 1.0.0 after feedback from the community. 694 | -------------------------------------------------------------------------------- /phi/builder.py: -------------------------------------------------------------------------------- 1 | """ 2 | The `phi.builder.Builder` extends the [Expression](https://cgarciae.github.io/phi/dsl.m.html#phi.dsl.Expression) class and adds methods that enable you to register external functions as methods. You should NOT register functions on the `Builder` or [PythonBuilder](https://cgarciae.github.io/phi/python_builder.m.html) classes, instead you should create a class of your own that extends these and register your functions there. The benefit of having this registration mechanism, instead of just implementing the methods yourself is that you can automate the process to register a collection of existing functions or methods. 3 | 4 | **See** 5 | 6 | * `phi.builder.Builder.RegisterMethod` 7 | * `phi.builder.Builder.RegisterAt` 8 | * `phi.builder.Builder.PatchAt` 9 | """ 10 | from __future__ import absolute_import 11 | from __future__ import division 12 | from __future__ import print_function 13 | from __future__ import unicode_literals 14 | 15 | 16 | import inspect 17 | from . import utils 18 | from .utils import identity 19 | import functools 20 | from . import dsl 21 | 22 | ###################### 23 | # Helpers 24 | ###################### 25 | 26 | _True = lambda x: True 27 | _False = lambda x: False 28 | _None = lambda x: None 29 | _NoLeadingUnderscore = lambda name: name[0] != "_" 30 | 31 | ####################### 32 | ### Builder 33 | ####################### 34 | 35 | class Builder(dsl.Expression): 36 | """ 37 | All the public methods of the `Builder`, `Expression` and `Expression` classes start with a capital letter on purpose to avoid name chashes with methods that you might register.""" 38 | 39 | @classmethod 40 | def _RegisterMethod(cls, f, library_path, alias=None, original_name=None, doc=None, wrapped=None, explanation="", method_type=utils.identity, explain=True): 41 | if wrapped: 42 | f = functools.wraps(wrapped)(f) 43 | 44 | fn_signature = utils.get_method_sig(f) 45 | fn_docs = inspect.getdoc(f) 46 | name = alias if alias else f.__name__ 47 | original_name = f.__name__ if wrapped else original_name if original_name else name 48 | 49 | f.__name__ = str(name) 50 | f.__doc__ = doc if doc else (""" 51 | THIS METHOD IS AUTOMATICALLY GENERATED 52 | 53 | {builder_class}.{name}(*args, **kwargs) 54 | 55 | It accepts the same arguments as `{library_path}{original_name}`. """ + explanation + """ 56 | 57 | **{library_path}{original_name}** 58 | 59 | {fn_docs} 60 | 61 | """).format(original_name=original_name, name=name, fn_docs=fn_docs, library_path=library_path, builder_class=cls.__name__) if explain else fn_docs 62 | 63 | if name in cls.__core__: 64 | raise Exception("Can't add method '{0}' because its on __core__".format(name)) 65 | 66 | f = method_type(f) 67 | setattr(cls, name, f) 68 | 69 | 70 | @classmethod 71 | def RegisterMethod(cls, *args, **kwargs): 72 | """ 73 | **RegisterMethod** 74 | 75 | RegisterMethod(f, library_path, alias=None, original_name=None, doc=None, wrapped=None, explanation="", method_type=utils.identity, explain=True) 76 | 77 | `classmethod` for registering functions as methods of this class. 78 | 79 | **Arguments** 80 | 81 | * **f** : the particular function being registered as a method 82 | * **library_path** : library from where `f` comes from, unless you pass an empty string, put a period `"."` at the end of the library name. 83 | * `alias=None` : alias for the name/method being registered 84 | * `original_name=None` : name of the original function, used for documentation purposes. 85 | * `doc=None` : complete documentation of the method being registered 86 | * `wrapped=None` : if you are registering a function which wraps around another function, pass this other function through `wrapped` to get better documentation, this is specially useful is you register a bunch of functions in a for loop. Please include an `explanation` to tell how the actual function differs from the wrapped one. 87 | * `explanation=""` : especify any additional information for the documentation of the method being registered, you can use any of the following format tags within this string and they will be replace latter on: `{original_name}`, `{name}`, `{fn_docs}`, `{library_path}`, `{builder_class}`. 88 | * `method_type=identity` : by default its applied but does nothing, you might also want to register functions as `property`, `classmethod`, `staticmethod` 89 | * `explain=True` : decide whether or not to show any kind of explanation, its useful to set it to `False` if you are using a `Register*` decorator and will only use the function as a registered method. 90 | 91 | A main feature of `phi` is that it enables you to integrate your library or even an existing library with the DSL. You can achieve three levels of integration 92 | 93 | 1. Passing your functions to the DSL. This a very general machanism -since you could actually do everything with python lamdas- but in practice functions often receive multiple parameters. 94 | 2. Creating partials with the `Then*` method family. Using this you could integrate any function, but it will add a lot of noise if you use heavily on it. 95 | 3. Registering functions as methods of a `Builder` derived class. This produces the most readable code and its the approach you should take if you want to create a Phi-based library or a helper class. 96 | 97 | While point 3 is the most desirable it has a cost: you need to create your own `phi.builder.Builder`-derived class. This is because SHOULD NOT register functions to existing builders e.g. the `phi.builder.Builder` or [PythonBuilder](https://cgarciae.github.io/phi/builder.m.html#phi.python_builder.PythonBuilder) provided by phi because that would pollute the `P` object. Instead you should create a custom class that derives from `phi.builder.Builder`, [PythonBuilder](https://cgarciae.github.io/phi/builder.m.html#phi.python_builder.PythonBuilder) or another custom builder depending on your needs and register your functions to that class. 98 | 99 | **Examples** 100 | 101 | Say you have a function on a library called `"my_lib"` 102 | 103 | def some_fun(obj, arg1, arg2): 104 | # code 105 | 106 | You could use it with the dsl like this 107 | 108 | from phi import P, Then 109 | 110 | P.Pipe( 111 | input, 112 | ... 113 | Then(some_fun, arg1, arg2) 114 | ... 115 | ) 116 | 117 | assuming the first parameter `obj` is being piped down. However if you do this very often or you are creating a library, you are better off creating a custom class derived from `Builder` or `PythonBuilder` 118 | 119 | from phi import Builder #or PythonBuilder 120 | 121 | class MyBuilder(Builder): # or PythonBuilder 122 | pass 123 | 124 | and registering your function as a method. The first way you could do this is by creating a wrapper function for `some_fun` and registering it as a method 125 | 126 | def some_fun_wrapper(self, arg1, arg2): 127 | return self.Then(some_fun, arg1, arg2) 128 | 129 | MyBuilder.RegisterMethod(some_fun_wrapper, "my_lib.", wrapped=some_fun) 130 | 131 | Here we basically created a shortcut for the original expression `Then(some_fun, arg1, arg2)`. You could also do this using a decorator 132 | 133 | @MyBuilder.RegisterMethod("my_lib.", wrapped=some_fun) 134 | def some_fun_wrapper(self, arg1, arg2): 135 | return self.Then(some_fun, arg1, arg2) 136 | 137 | However, this is such a common task that we've created the method `Register` to avoid you from having to create the wrapper. With it you could register the function `some_fun` directly as a method like this 138 | 139 | MyBuilder.Register(some_fun, "my_lib.") 140 | 141 | or by using a decorator over the original function definition 142 | 143 | @MyBuilder.Register("my_lib.") 144 | def some_fun(obj, arg1, arg2): 145 | # code 146 | 147 | Once done you've done any of the previous approaches you can create a custom global object e.g. `M` and use it instead of/along with `P` 148 | 149 | M = MyBuilder(lambda x: x) 150 | 151 | M.Pipe( 152 | input, 153 | ... 154 | M.some_fun(arg1, args) 155 | ... 156 | ) 157 | 158 | **Argument position** 159 | 160 | `phi.builder.Builder.Register` internally uses `phi.builder.Builder.Then`, this is only useful if the object being piped is intended to be passed as the first argument of the function being registered, if this is not the case you could use `phi.builder.Builder.Register2`, `phi.builder.Builder.Register3`, ..., `phi.builder.Builder.Register5` or `phi.builder.Builder.RegisterAt` to set an arbitrary position, these functions will internally use `phi.builder.Builder.Then2`, `phi.builder.Builder.Then3`, ..., `phi.builder.Builder.Then5` or `phi.builder.Builder.ThenAt` respectively. 161 | 162 | **Wrapping functions** 163 | 164 | Sometimes you have an existing function that you would like to modify slightly so it plays nicely with the DSL, what you normally do is create a function that wraps around it and passes the arguments to it in a way that is convenient 165 | 166 | import some_lib 167 | 168 | @MyBuilder.Register("some_lib.") 169 | def some_fun(a, n): 170 | return some_lib.some_fun(a, n - 1) # forward the args, n slightly modified 171 | 172 | When you do this -as a side effect- you loose the original documentation, to avoid this you can use the Registers `wrapped` argument along with the `explanation` argument to clarity the situation 173 | 174 | import some_lib 175 | 176 | some_fun_explanation = "However, it differs in that `n` is automatically subtracted `1`" 177 | 178 | @MyBuilder.Register("some_lib.", wrapped=some_lib.some_fun, explanation=some_fun_explanation) 179 | def some_fun(a, n): 180 | return some_lib.some_fun(a, n - 1) # forward the args, n slightly modified 181 | 182 | Now the documentation for `MyBuilder.some_fun` will be a little bit nicer since it includes the original documentation from `some_lib.some_fun`. This behaviour is specially useful if you are wrapping an entire 3rd party library, you usually automate the process iterating over all the funcitions in a for loop. The `phi.builder.Builder.PatchAt` method lets you register and entire module using a few lines of code, however, something you have to do thing more manually and do the iteration yourself. 183 | 184 | **See Also** 185 | 186 | * `phi.builder.Builder.PatchAt` 187 | * `phi.builder.Builder.RegisterAt` 188 | """ 189 | unpack_error = True 190 | 191 | try: 192 | f, library_path = args 193 | unpack_error = False 194 | cls._RegisterMethod(f, library_path, **kwargs) 195 | 196 | except: 197 | if not unpack_error: 198 | raise 199 | 200 | def register_decorator(f): 201 | library_path, = args 202 | cls._RegisterMethod(f, library_path, **kwargs) 203 | 204 | return f 205 | return register_decorator 206 | 207 | 208 | @classmethod 209 | def _RegisterAt(cls, n, f, library_path, alias=None, original_name=None, doc=None, wrapped=None, explanation="", method_type=utils.identity, explain=True, _return_type=None): 210 | 211 | _wrapped = wrapped if wrapped else f 212 | 213 | try: 214 | @functools.wraps(f) 215 | def method(self, *args, **kwargs): 216 | 217 | kwargs['_return_type'] = _return_type 218 | return self.ThenAt(n, f, *args, **kwargs) 219 | except: 220 | raise 221 | 222 | all_args, previous_args, last_arg = _make_args_strs(n) 223 | 224 | explanation = """ 225 | However, the 1st argument is omitted, a partial with the rest of the arguments is returned which expects the 1st argument such that 226 | 227 | {library_path}{original_name}("""+ all_args +"""*args, **kwargs) 228 | 229 | is equivalent to 230 | 231 | {builder_class}.{name}("""+ previous_args +"""*args, **kwargs)("""+ last_arg +""") 232 | 233 | """ + explanation if explain else "" 234 | 235 | cls.RegisterMethod(method, library_path, alias=alias, original_name=original_name, doc=doc, wrapped=wrapped, explanation=explanation, method_type=method_type, explain=explain) 236 | 237 | @classmethod 238 | def RegisterAt(cls, *args, **kwargs): 239 | """ 240 | **RegisterAt** 241 | 242 | RegisterAt(n, f, library_path, alias=None, original_name=None, doc=None, wrapped=None, explanation="", method_type=utils.identity, explain=True, _return_type=None) 243 | 244 | Most of the time you don't want to register an method as such, that is, you don't care about the `self` builder object, instead you want to register a function that transforms the value being piped down the DSL. For this you can use `RegisterAt` so e.g. 245 | 246 | def some_fun(obj, arg1, arg2): 247 | # code 248 | 249 | @MyBuilder.RegisterMethod("my_lib.") 250 | def some_fun_wrapper(self, arg1, arg2): 251 | return self.ThenAt(1, some_fun, arg1, arg2) 252 | 253 | can be written directly as 254 | 255 | @MyBuilder.RegisterAt(1, "my_lib.") 256 | def some_fun(obj, arg1, arg2): 257 | # code 258 | 259 | For this case you can just use `Register` which is a shortcut for `RegisterAt(1, ...)` 260 | 261 | @MyBuilder.Register("my_lib.") 262 | def some_fun(obj, arg1, arg2): 263 | # code 264 | 265 | **Also See** 266 | 267 | * `phi.builder.Builder.RegisterMethod` 268 | """ 269 | unpack_error = True 270 | 271 | try: 272 | n, f, library_path = args 273 | unpack_error = False 274 | cls._RegisterAt(n, f, library_path, **kwargs) 275 | 276 | except: 277 | if not unpack_error: 278 | raise 279 | 280 | def register_decorator(f): 281 | n, library_path = args 282 | cls._RegisterAt(n, f, library_path, **kwargs) 283 | 284 | return f 285 | return register_decorator 286 | 287 | @classmethod 288 | def Register0(cls, *args, **kwargs): 289 | """ 290 | `Register0(...)` is a shortcut for `RegisterAt(0, ...)` 291 | 292 | **Also See** 293 | 294 | * `phi.builder.Builder.RegisterAt` 295 | * `phi.builder.Builder.RegisterMethod` 296 | """ 297 | return cls.RegisterAt(0, *args, **kwargs) 298 | 299 | @classmethod 300 | def Register(cls, *args, **kwargs): 301 | """ 302 | `Register(...)` is a shortcut for `RegisterAt(1, ...)` 303 | 304 | **Also See** 305 | 306 | * `phi.builder.Builder.RegisterAt` 307 | * `phi.builder.Builder.RegisterMethod` 308 | """ 309 | return cls.RegisterAt(1, *args, **kwargs) 310 | 311 | @classmethod 312 | def Register2(cls, *args, **kwargs): 313 | """ 314 | `Register2(...)` is a shortcut for `RegisterAt(2, ...)` 315 | 316 | **Also See** 317 | 318 | * `phi.builder.Builder.RegisterAt` 319 | * `phi.builder.Builder.RegisterMethod` 320 | """ 321 | return cls.RegisterAt(2, *args, **kwargs) 322 | 323 | @classmethod 324 | def Register3(cls, *args, **kwargs): 325 | """ 326 | `Register3(...)` is a shortcut for `RegisterAt(3, ...)` 327 | 328 | **Also See** 329 | 330 | * `phi.builder.Builder.RegisterAt` 331 | * `phi.builder.Builder.RegisterMethod` 332 | """ 333 | return cls.RegisterAt(3, *args, **kwargs) 334 | 335 | @classmethod 336 | def Register4(cls, *args, **kwargs): 337 | """ 338 | `Register4(...)` is a shortcut for `RegisterAt(4, ...)` 339 | 340 | **Also See** 341 | 342 | * `phi.builder.Builder.RegisterAt` 343 | * `phi.builder.Builder.RegisterMethod` 344 | """ 345 | return cls.RegisterAt(4, *args, **kwargs) 346 | 347 | @classmethod 348 | def Register5(cls, *args, **kwargs): 349 | """ 350 | `Register5(...)` is a shortcut for `RegisterAt(5, ...)` 351 | 352 | **Also See** 353 | 354 | * `phi.builder.Builder.RegisterAt` 355 | * `phi.builder.Builder.RegisterMethod` 356 | """ 357 | return cls.RegisterAt(5, *args, **kwargs) 358 | 359 | @classmethod 360 | def PatchAt(cls, n, module, method_wrapper=None, module_alias=None, method_name_modifier=utils.identity, blacklist_predicate=_False, whitelist_predicate=_True, return_type_predicate=_None, getmembers_predicate=inspect.isfunction, admit_private=False, explanation=""): 361 | """ 362 | This classmethod lets you easily patch all of functions/callables from a module or class as methods a Builder class. 363 | 364 | **Arguments** 365 | 366 | * **n** : the position the the object being piped will take in the arguments when the function being patched is applied. See `RegisterMethod` and `ThenAt`. 367 | * **module** : a module or class from which the functions/methods/callables will be taken. 368 | * `module_alias = None` : an optional alias for the module used for documentation purposes. 369 | * `method_name_modifier = lambda f_name: None` : a function that can modify the name of the method will take. If `None` the name of the function will be used. 370 | * `blacklist_predicate = lambda f_name: name[0] != "_"` : A predicate that determines which functions are banned given their name. By default it excludes all function whose name start with `'_'`. `blacklist_predicate` can also be of type list, in which case all names contained in this list will be banned. 371 | * `whitelist_predicate = lambda f_name: True` : A predicate that determines which functions are admitted given their name. By default it include any function. `whitelist_predicate` can also be of type list, in which case only names contained in this list will be admitted. You can use both `blacklist_predicate` and `whitelist_predicate` at the same time. 372 | * `return_type_predicate = lambda f_name: None` : a predicate that determines the `_return_type` of the Builder. By default it will always return `None`. See `phi.builder.Builder.ThenAt`. 373 | * `getmembers_predicate = inspect.isfunction` : a predicate that determines what type of elements/members will be fetched by the `inspect` module, defaults to [inspect.isfunction](https://docs.python.org/2/library/inspect.html#inspect.isfunction). See [getmembers](https://docs.python.org/2/library/inspect.html#inspect.getmembers). 374 | 375 | **Examples** 376 | 377 | Lets patch ALL the main functions from numpy into a custom builder! 378 | 379 | from phi import PythonBuilder #or Builder 380 | import numpy as np 381 | 382 | class NumpyBuilder(PythonBuilder): #or Builder 383 | "A Builder for numpy functions!" 384 | pass 385 | 386 | NumpyBuilder.PatchAt(1, np) 387 | 388 | N = NumpyBuilder(lambda x: x) 389 | 390 | Thats it! Although a serious patch would involve filtering out functions that don't take arrays. Another common task would be to use `NumpyBuilder.PatchAt(2, ...)` (`PatchAt(n, ..)` in general) when convenient to send the object being pipe to the relevant argument of the function. The previous is usually done with and a combination of `whitelist_predicate`s and `blacklist_predicate`s on `PatchAt(1, ...)` and `PatchAt(2, ...)` to filter or include the approriate functions on each kind of patch. Given the previous code we could now do 391 | 392 | import numpy as np 393 | 394 | x = np.array([[1,2],[3,4]]) 395 | y = np.array([[5,6],[7,8]]) 396 | 397 | z = N.Pipe( 398 | x, N 399 | .dot(y) 400 | .add(x) 401 | .transpose() 402 | .sum(axis=1) 403 | ) 404 | 405 | Which is strictly equivalent to 406 | 407 | import numpy as np 408 | 409 | x = np.array([[1,2],[3,4]]) 410 | y = np.array([[5,6],[7,8]]) 411 | 412 | z = np.dot(x, y) 413 | z = np.add(z, x) 414 | z = np.transpose(z) 415 | z = np.sum(z, axis=1) 416 | 417 | The thing to notice is that with the `NumpyBuilder` we avoid the repetitive and needless passing and reassigment of the `z` variable, this removes a lot of noise from our code. 418 | """ 419 | _rtp = return_type_predicate 420 | 421 | return_type_predicate = (lambda x: _rtp) if inspect.isclass(_rtp) and issubclass(_rtp, Builder) else _rtp 422 | module_name = module_alias if module_alias else module.__name__ + '.' 423 | patch_members = _get_patch_members(module, blacklist_predicate=blacklist_predicate, whitelist_predicate=whitelist_predicate, getmembers_predicate=getmembers_predicate, admit_private=admit_private) 424 | 425 | for name, f in patch_members: 426 | wrapped = None 427 | 428 | if method_wrapper: 429 | g = method_wrapper(f) 430 | wrapped = f 431 | else: 432 | g = f 433 | 434 | cls.RegisterAt(n, g, module_name, wrapped=wrapped, _return_type=return_type_predicate(name), alias=method_name_modifier(name), explanation=explanation) 435 | 436 | 437 | Builder.__core__ = [ name for name, f in inspect.getmembers(Builder, inspect.ismethod) ] 438 | 439 | 440 | 441 | ####################### 442 | # Helper functions 443 | ####################### 444 | 445 | def _make_args_strs(n): 446 | 447 | if n == 0: 448 | return "", "", "x" 449 | 450 | n += 1 451 | all_args = [ 'x' + str(i) for i in range(1, n) ] 452 | last = all_args[n-2] 453 | previous = all_args[:n-2] 454 | 455 | return ", ".join(all_args + [""]), ", ".join(previous + [""]), last 456 | 457 | def _get_patch_members(module, blacklist_predicate=_NoLeadingUnderscore, whitelist_predicate=_True, _return_type=None, getmembers_predicate=inspect.isfunction, admit_private=False): 458 | 459 | if type(whitelist_predicate) is list: 460 | whitelist = whitelist_predicate 461 | whitelist_predicate = lambda x: x in whitelist 462 | 463 | if type(blacklist_predicate) is list: 464 | blacklist = blacklist_predicate 465 | blacklist_predicate = lambda x: x in blacklist or '_' == x[0] if not admit_private else False 466 | 467 | return [ 468 | (name, f) for (name, f) in inspect.getmembers(module, getmembers_predicate) if whitelist_predicate(name) and not blacklist_predicate(name) 469 | ] 470 | -------------------------------------------------------------------------------- /docs/phi/tests/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | phi.tests API documentation 7 | 8 | 9 | 10 | 11 | 551 | 552 | 853 | 854 | 921 | 922 | 1014 | 1015 | 1029 | 1030 | 1031 | Top 1032 | 1033 |
1034 | 1035 | 1036 | 1056 | 1057 |
1058 | 1059 | 1060 | 1061 | 1062 | 1063 | 1064 |
1065 |

phi.tests module

1066 | 1067 | 1068 | 1069 |
1070 | 1071 |
1072 | 1073 | 1074 | 1075 |

Sub-modules

1076 |
1077 |

phi.tests.some_module

1078 | 1079 | 1080 | 1081 |
1082 |
1083 |

phi.tests.test_builders

1084 | 1085 | 1086 | 1087 |
1088 |
1089 |

phi.tests.test_dsl

1090 | 1091 | 1092 | 1093 |
1094 |
1095 |

phi.tests.test_dummy

1096 | 1097 | 1098 | 1099 |
1100 |
1101 |

phi.tests.test_examples

1102 | 1103 | 1104 | 1105 |
1106 |
1107 |

phi.tests.test_lambdas

1108 | 1109 | 1110 | 1111 |
1112 |
1113 |

phi.tests.test_nodsl

1114 | 1115 | 1116 | 1117 |
1118 |
1119 |

phi.tests.test_state

1120 | 1121 | 1122 | 1123 |
1124 |
1125 | 1126 |
1127 |
1128 | 1139 |
1140 | 1141 | 1142 | -------------------------------------------------------------------------------- /docs/phi/tests/some_module.m.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | phi.tests.some_module API documentation 7 | 8 | 9 | 10 | 11 | 551 | 552 | 853 | 854 | 921 | 922 | 1014 | 1015 | 1029 | 1030 | 1031 | Top 1032 | 1033 |
1034 | 1035 | 1036 | 1052 | 1053 |
1054 | 1055 | 1056 | 1057 | 1058 | 1059 | 1060 |
1061 |

phi.tests.some_module module

1062 | 1063 | 1064 |

Show source ≡

1065 |
1066 | 
ef some_fun(x):
1067 |    return "YES"
1068 | ef other_fun(x, y):
1069 |    return x / y
1070 |
1071 | 1072 |
1073 | 1074 |
1075 | 1076 |
1077 | 1078 |

Functions

1079 | 1080 |
1081 |
1082 |

def other_fun(

x, y)

1083 |
1084 | 1085 | 1086 | 1087 | 1088 |
1089 |

Show source ≡

1090 |
1091 | 
def other_fun(x, y):
1092 |     return x / y
1093 |
1094 | 1095 |
1096 |
1097 | 1098 |
1099 | 1100 | 1101 |
1102 |
1103 |

def some_fun(

x)

1104 |
1105 | 1106 | 1107 | 1108 | 1109 |
1110 |

Show source ≡

1111 |
1112 | 
def some_fun(x):
1113 |     return "YES"
1114 |
1115 | 1116 |
1117 |
1118 | 1119 |
1120 | 1121 | 1122 | 1123 |
1124 | 1125 |
1126 |
1127 | 1138 |
1139 | 1140 | 1141 | --------------------------------------------------------------------------------