├── .github └── workflows │ └── pythonapp.yml ├── .gitignore ├── .readthedocs.yml ├── README.md ├── README.rst ├── docs ├── Makefile ├── make.bat └── source │ ├── conf.py │ ├── fn_graph.rst │ ├── index.rst │ ├── intro.gv.png │ ├── linked_namespaces.png │ ├── modules.rst │ ├── namespaces.png │ ├── readme.md │ ├── requirements.txt │ └── usage.md ├── fn_graph ├── __init__.py ├── caches.py ├── calculation.py ├── examples │ ├── __init__.py │ ├── broken.py │ ├── caching.py │ ├── car_savings.py │ ├── credit.py │ ├── credit_data.csv │ ├── finance.py │ ├── machine_learning.py │ ├── namespaces.py │ ├── plotting.py │ ├── share_prices.csv │ ├── shares_in_issue.csv │ └── stock_market.py ├── profiler.py ├── tests │ ├── __init__.py │ ├── large_graph.py │ ├── test_basics.py │ ├── test_cache_methods.py │ ├── test_links.py │ └── utils.py └── usage.py ├── intro.gv.png ├── linked_namespaces.png ├── poetry.lock ├── pyproject.toml └── setup.py /.github/workflows/pythonapp.yml: -------------------------------------------------------------------------------- 1 | name: Python application 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@v1 12 | - name: Set up Python 3.7 13 | uses: actions/setup-python@v1 14 | with: 15 | python-version: 3.7 16 | - name: Install dependencies 17 | run: | 18 | python -m pip install --upgrade pip poetry 19 | poetry config virtualenvs.create false --local 20 | poetry install 21 | - name: Test with pytest 22 | run: | 23 | pytest 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | db.sqlite3-journal 62 | 63 | # Flask stuff: 64 | instance/ 65 | .webassets-cache 66 | 67 | # Scrapy stuff: 68 | .scrapy 69 | 70 | # Sphinx documentation 71 | docs/_build/ 72 | 73 | # PyBuilder 74 | target/ 75 | 76 | # Jupyter Notebook 77 | .ipynb_checkpoints 78 | 79 | # IPython 80 | profile_default/ 81 | ipython_config.py 82 | 83 | # pyenv 84 | .python-version 85 | 86 | # pipenv 87 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 88 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 89 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 90 | # install all needed dependencies. 91 | #Pipfile.lock 92 | 93 | # celery beat schedule file 94 | celerybeat-schedule 95 | 96 | # SageMath parsed files 97 | *.sage.py 98 | 99 | # Environments 100 | .env 101 | .venv 102 | env/ 103 | venv/ 104 | ENV/ 105 | env.bak/ 106 | venv.bak/ 107 | 108 | # Spyder project settings 109 | .spyderproject 110 | .spyproject 111 | 112 | # Rope project settings 113 | .ropeproject 114 | 115 | # mkdocs documentation 116 | /site 117 | 118 | # mypy 119 | .mypy_cache/ 120 | .dmypy.json 121 | dmypy.json 122 | 123 | # Pyre type checker 124 | .pyre/ 125 | 126 | # VSCode 127 | .vscode 128 | 129 | # CSS 130 | *.css 131 | *.css.map 132 | 133 | # Data folder 134 | data/ 135 | test_data/ 136 | 137 | # graphviz 138 | Source.gv 139 | Source.gv.pdf 140 | Digraph.gv 141 | Digraph.gv.pdf 142 | 143 | # Fn_graph cache 144 | .fn_graph_cache 145 | 146 | # Sandbox files for developers 147 | sandbox.py -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Build documentation in the docs/ directory with Sphinx 9 | sphinx: 10 | configuration: docs/source/conf.py 11 | 12 | # Optionally build your docs in additional formats such as PDF and ePub 13 | formats: all 14 | 15 | # Optionally set the version of Python and requirements required to build your docs 16 | python: 17 | version: 3.7 18 | install: 19 | - requirements: docs/source/requirements.txt 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Fn Graph 2 | 3 | Lightweight function pipelines for python 4 | 5 | For more information and live examples look at [fn-graph.businessoptics.biz](https://fn-graph.businessoptics.biz/) 6 | 7 | ## Overview 8 | 9 | `fn_graph` is trying to solve a number of problems in the python data-science/modelling domain, as well as making it easier to put such models into production. 10 | 11 | It aims to: 12 | 13 | 1. Make moving between the analyst space to production, and back, simpler and less error prone. 14 | 2. Make it easy to view the intermediate results of computations to easily diagnose errors. 15 | 3. Solve common analyst issues like creating reusable, composable pipelines and caching results. 16 | 4. Visualizing models in an intuitive way. 17 | 18 | There is an associated visual studio you should check out at https://github.com/BusinessOptics/fn_graph_studio/. 19 | 20 | ## Documentation 21 | 22 | Please find detailed documentation at https://fn-graph.readthedocs.io/ 23 | 24 | ## Installation 25 | 26 | ```sh 27 | pip install fn_graph 28 | ``` 29 | 30 | You will need to have graphviz and the development packages installed. On ubuntu you can install these with: 31 | 32 | ```sh 33 | sudo apt-get install graphviz graphviz-dev 34 | ``` 35 | 36 | Otherwise see the [pygraphviz documentation](http://pygraphviz.github.io/documentation/pygraphviz-1.5/install.html). 37 | 38 | To run all the examples install 39 | 40 | ```sh 41 | pip install fn_graph[examples] 42 | ``` 43 | 44 | ## Features 45 | 46 | * **Manage complex logic**\ 47 | Manage your data processing, machine learning, domain or financial logic all in one simple unified framework. Make models that are easy to understand at a meaningful level of abstraction. 48 | * **Hassle free moves to production**\ 49 | Take the models your data-scientist and analysts build and move them into your production environment, whether thats a task runner, web-application, or an API. No recoding, no wrapping notebook code in massive and opaque functions. When analysts need to make changes they can easily investigate all the models steps. 50 | * **Lightweight**\ 51 | Fn Graph is extremely minimal. Develop your model as plain python functions and it will connect everything together. There is no complex object model to learn or heavy weight framework code to manage. 52 | * **Visual model explorer**\ 53 | Easily navigate and investigate your models with the visual fn_graph_studio. Share knowledge amongst your team and with all stakeholders. Quickly isolate interesting results or problematic errors. Visually display your results with any popular plotting libraries. 54 | * **Work with or without notebooks**\ 55 | Use fn_graph as a complement to your notebooks, or use it with your standard development tools, or both. 56 | 57 | * **Works with whatever libraries you use**\ 58 | fn_graph makes no assumptions about what libraries you use. Use your favorite machine learning libraries like, scikit-learn, PyTorch. Prepare your data with data with Pandas or Numpy. Crunch big data with PySpark or Beam. Plot results with matplotlib, seaborn or Plotly. Use statistical routines from Scipy or your favourite financial libraries. Or just use plain old Python, it's up to you. 59 | * **Useful modelling support tools**\ 60 | Integrated and intelligent caching improves modelling development iteration time, a simple profiler works at a level that's meaningful to your model. 61 | ** *Easily compose and reuse models**\ 62 | The composable pipelines allow for easy model reuse, as well as building up models from simpler submodels. Easily collaborate in teams to build models to any level of complexity, while keeping the individual components easy to understand and well encapsulated. 63 | * **It's just Python functions**\ 64 | It's just plain Python! Use all your existing knowledge, everything will work as expected. Integrate with any existing python codebases. Use it with any other framework, there are no restrictions. 65 | 66 | ## Similar projects 67 | 68 | An incomplete comparison to some other libraries, highlighting the differences: 69 | 70 | **Dask** 71 | 72 | Dask is a light-weight parallel computing library. Importantly it has a Pandas compliant interface. You may want to use Dask inside FnGraph. 73 | 74 | **Airflow** 75 | 76 | Airflow is a task manager. It is used to run a series of generally large tasks in an order that meets their dependencies, potentially over multiple machines. It has a whole scheduling and management apparatus around it. Fn Graph is not trying to do this. Fn Graph is about making complex logic more manageable, and easier to move between development and production. You may well want to use Fn Graph inside your airflow tasks. 77 | 78 | **Luigi** 79 | 80 | > Luigi is a Python module that helps you build complex pipelines of batch jobs. It handles dependency resolution, workflow management, visualization etc. It also comes with Hadoop support built in. 81 | 82 | Luigi is about big batch jobs, and managing the distribution and scheduling of them. In the same way that airflow works ate a higher level to FnGraph, so does luigi. 83 | 84 | **d6tflow** 85 | 86 | d6tflow is similar to FnGraph. It is based on Luigi. The primary difference is the way the function graphs are composed. d6tflow graphs can be very difficult to reuse (but do have some greater flexibility). It also allows for parallel execution. FnGraph is trying to make very complex pipelines or very complex models easier to mange, build, and productionise. 87 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | 2 | Fn Graph 3 | ======== 4 | 5 | Lightweight function pipelines for python 6 | 7 | For more information and live examples look at `fn-graph.businessoptics.biz `_ 8 | 9 | Overview 10 | -------- 11 | 12 | ``fn_graph`` is trying to solve a number of problems in the python data-science/modelling domain, as well as making it easier to put such models into production. 13 | 14 | It aims to: 15 | 16 | 17 | #. Make moving between the analyst space to production, and back, simpler and less error prone. 18 | #. Make it easy to view the intermediate results of computations to easily diagnose errors. 19 | #. Solve common analyst issues like creating reusable, composable pipelines and caching results. 20 | #. Visualizing models in an intuitive way. 21 | 22 | There is an associated visual studio you should check out at https://github.com/BusinessOptics/fn_graph_studio/. 23 | 24 | Documentation 25 | ------------- 26 | 27 | Please find detailed documentation at https://fn-graph.readthedocs.io/ 28 | 29 | Installation 30 | ------------ 31 | 32 | .. code-block:: sh 33 | 34 | pip install fn_graph 35 | 36 | You will need to have graphviz and the development packages installed. On ubuntu you can install these with: 37 | 38 | .. code-block:: sh 39 | 40 | sudo apt-get install graphviz graphviz-dev 41 | 42 | Otherwise see the `pygraphviz documentation `_. 43 | 44 | To run all the examples install 45 | 46 | .. code-block:: sh 47 | 48 | pip install fn_graph[examples] 49 | 50 | Features 51 | -------- 52 | 53 | 54 | * **Manage complex logic**\ \ 55 | Manage your data processing, machine learning, domain or financial logic all in one simple unified framework. Make models that are easy to understand at a meaningful level of abstraction. 56 | * **Hassle free moves to production**\ \ 57 | Take the models your data-scientist and analysts build and move them into your production environment, whether thats a task runner, web-application, or an API. No recoding, no wrapping notebook code in massive and opaque functions. When analysts need to make changes they can easily investigate all the models steps. 58 | * **Lightweight**\ \ 59 | Fn Graph is extremely minimal. Develop your model as plain python functions and it will connect everything together. There is no complex object model to learn or heavy weight framework code to manage. 60 | * **Visual model explorer**\ \ 61 | Easily navigate and investigate your models with the visual fn_graph_studio. Share knowledge amongst your team and with all stakeholders. Quickly isolate interesting results or problematic errors. Visually display your results with any popular plotting libraries. 62 | * 63 | **Work with or without notebooks**\ \ 64 | Use fn_graph as a complement to your notebooks, or use it with your standard development tools, or both. 65 | 66 | * 67 | **Works with whatever libraries you use**\ \ 68 | fn_graph makes no assumptions about what libraries you use. Use your favorite machine learning libraries like, scikit-learn, PyTorch. Prepare your data with data with Pandas or Numpy. Crunch big data with PySpark or Beam. Plot results with matplotlib, seaborn or Plotly. Use statistical routines from Scipy or your favourite financial libraries. Or just use plain old Python, it's up to you. 69 | 70 | * **Useful modelling support tools**\ \ 71 | Integrated and intelligent caching improves modelling development iteration time, a simple profiler works at a level that's meaningful to your model. 72 | ** *Easily compose and reuse models**\ \ 73 | The composable pipelines allow for easy model reuse, as well as building up models from simpler submodels. Easily collaborate in teams to build models to any level of complexity, while keeping the individual components easy to understand and well encapsulated. 74 | * **It's just Python functions**\ \ 75 | It's just plain Python! Use all your existing knowledge, everything will work as expected. Integrate with any existing python codebases. Use it with any other framework, there are no restrictions. 76 | 77 | Similar projects 78 | ---------------- 79 | 80 | An incomplete comparison to some other libraries, highlighting the differences: 81 | 82 | **Dask** 83 | 84 | Dask is a light-weight parallel computing library. Importantly it has a Pandas compliant interface. You may want to use Dask inside FnGraph. 85 | 86 | **Airflow** 87 | 88 | Airflow is a task manager. It is used to run a series of generally large tasks in an order that meets their dependencies, potentially over multiple machines. It has a whole scheduling and management apparatus around it. Fn Graph is not trying to do this. Fn Graph is about making complex logic more manageable, and easier to move between development and production. You may well want to use Fn Graph inside your airflow tasks. 89 | 90 | **Luigi** 91 | 92 | .. 93 | 94 | Luigi is a Python module that helps you build complex pipelines of batch jobs. It handles dependency resolution, workflow management, visualization etc. It also comes with Hadoop support built in. 95 | 96 | 97 | Luigi is about big batch jobs, and managing the distribution and scheduling of them. In the same way that airflow works ate a higher level to FnGraph, so does luigi. 98 | 99 | **d6tflow** 100 | 101 | d6tflow is similar to FnGraph. It is based on Luigi. The primary difference is the way the function graphs are composed. d6tflow graphs can be very difficult to reuse (but do have some greater flexibility). It also allows for parallel execution. FnGraph is trying to make very complex pipelines or very complex models easier to mange, build, and productionise. 102 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import sys 15 | 16 | sys.path.insert(0, os.path.abspath("../..")) 17 | 18 | 19 | # -- Project information ----------------------------------------------------- 20 | 21 | project = "Fn Graph" 22 | copyright = "2019, James Saunders" 23 | author = "James Saunders" 24 | 25 | 26 | # -- General configuration --------------------------------------------------- 27 | 28 | # Add any Sphinx extension module names here, as strings. They can be 29 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 30 | # ones. 31 | extensions = ["recommonmark", "sphinx.ext.napoleon"] 32 | 33 | 34 | # Add any paths that contain templates here, relative to this directory. 35 | templates_path = ["_templates"] 36 | 37 | # List of patterns, relative to source directory, that match files and 38 | # directories to ignore when looking for source files. 39 | # This pattern also affects html_static_path and html_extra_path. 40 | exclude_patterns = [] 41 | 42 | master_doc = "index" 43 | # -- Options for HTML output ------------------------------------------------- 44 | 45 | # The theme to use for HTML and HTML Help pages. See the documentation for 46 | # a list of builtin themes. 47 | # 48 | html_theme = "sphinx_rtd_theme" 49 | 50 | # Add any paths that contain custom static files (such as style sheets) here, 51 | # relative to this directory. They are copied after the builtin static files, 52 | # so a file named "default.css" will overwrite the builtin "default.css". 53 | html_static_path = ["_static"] 54 | 55 | -------------------------------------------------------------------------------- /docs/source/fn_graph.rst: -------------------------------------------------------------------------------- 1 | fn\_graph package 2 | ================= 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: fn_graph 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. Fn Graph documentation master file, created by 2 | sphinx-quickstart on Fri Dec 13 02:03:24 2019. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to Fn Graph's documentation! 7 | ==================================== 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | 12 | Introduction 13 | usage 14 | API Reference 15 | 16 | -------------------------------------------------------------------------------- /docs/source/intro.gv.png: -------------------------------------------------------------------------------- 1 | ../intro.gv.png -------------------------------------------------------------------------------- /docs/source/linked_namespaces.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BusinessOptics/fn_graph/50ef05ce07c069383850cbdba190af790278419b/docs/source/linked_namespaces.png -------------------------------------------------------------------------------- /docs/source/modules.rst: -------------------------------------------------------------------------------- 1 | fn_graph 2 | ======== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | fn_graph 8 | -------------------------------------------------------------------------------- /docs/source/namespaces.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BusinessOptics/fn_graph/50ef05ce07c069383850cbdba190af790278419b/docs/source/namespaces.png -------------------------------------------------------------------------------- /docs/source/readme.md: -------------------------------------------------------------------------------- 1 | ../../README.md -------------------------------------------------------------------------------- /docs/source/requirements.txt: -------------------------------------------------------------------------------- 1 | graphviz 2 | littleutils 3 | joblib 4 | networkx 5 | typing -------------------------------------------------------------------------------- /docs/source/usage.md: -------------------------------------------------------------------------------- 1 | # Usage 2 | 3 | ## Building up logic 4 | 5 | That principle idea behind Fn Graph is to use the names of a functions arguments to find that 6 | functions dependencies, and hence wire up the graph. 7 | 8 | There are multiple methods to add functions to the graph, all of them use the underlying 9 | `update` function. In the most direct form update takes keyword arguments, the keyword defines 10 | the name of the function in the graph. For example: 11 | 12 | ```python 13 | from fn_graph import Composer 14 | 15 | def get_a(): 16 | return 5 17 | 18 | def get_b(a): 19 | return a * 5 20 | 21 | def get_c(a, b): 22 | return a * b 23 | 24 | composer = Composer().update(a=get_a, b=get_b, c=get_c) 25 | ``` 26 | 27 | Alternatively, some may prefer the more terse version where the function name itself is used as 28 | the name within the graph, for example: 29 | 30 | ```python 31 | from fn_graph import Composer 32 | 33 | def a(): 34 | return 5 35 | 36 | def b(a): 37 | return a * 5 38 | 39 | def c(a, b): 40 | return a * b 41 | 42 | composer = Composer().update(a=a, b=b, c=c) 43 | ``` 44 | 45 | The issue with this is it leads to name shadowing, which some people don't like, and many linters 46 | will (rightly) complain about. An alternative that is reasonably terse and does not have the name 47 | shadowing problem is to use the prefix or suffix stripping versions of update. 48 | 49 | ```python 50 | from fn_graph import Composer 51 | 52 | def get_a(): 53 | return 5 54 | 55 | def get_b(a): 56 | return a * 5 57 | 58 | def get_c(a, b): 59 | return a * b 60 | 61 | composer = Composer().update_without_prefix("get_", get_a, get_b, get_c) 62 | ``` 63 | 64 | Often you have static inputs into a graph, parameters. It is more convenient ot treat these differently rather than creating functions that just return the values, and use the `update_parameters` method. 65 | 66 | ```python 67 | from fn_graph import Composer 68 | 69 | def get_b(a): 70 | return a * 5 71 | 72 | def get_c(a, b): 73 | return a * b 74 | 75 | composer = ( 76 | Composer() 77 | .update_without_prefix("get_", get_b, get_c) 78 | .update_parameters(a=5) 79 | ) 80 | ``` 81 | 82 | All update methods return a nw composer, and as such can be safely chained together. You can also update a composer with all the functions of another composer using the `update_from` method. 83 | 84 | ```python 85 | composer_a = ... 86 | 87 | composer_b = ... 88 | 89 | composer_c = composer_b.update_from(composer_a) 90 | ``` 91 | 92 | ## Visualisation 93 | 94 | You can see the function graph using the `graphviz` method. In a notebook environment this will be rendered directly. In other environment you may want to use the view method. 95 | 96 | ```python 97 | # In a notebbook 98 | composer.graphviz() 99 | 100 | #In other environments 101 | composer.graphviz().view() 102 | ``` 103 | 104 | ## Calculation 105 | 106 | The function graph can be calculated using the `calculate` method. It can calculate multiple results at once, and can return all the intermediate results. It returns a dictionary of the results 107 | 108 | ```python 109 | composer.calculate(["a" ,"c"]) // {"a": 5, "c": 125} 110 | composer.calculate(["a" ,"c"], intermediates=True) // {"a": 5, "b": 25, "c": 125} 111 | ``` 112 | 113 | You can als use the call function if you want only a single result. 114 | 115 | ```python 116 | composer.call("c") // 125 117 | ``` 118 | 119 | As an accelerator calling a function is exposed as a method on the composer. 120 | 121 | ```python 122 | composer.c() // 125 123 | ``` 124 | 125 | ## Caching 126 | 127 | Caching is provided primarily to make development easier. The development cache will cache results to disk, so it persists between sessions. Additionally it stores a hash of the various functions in the composer, and will invalidate the cache when a change is made. This works well as long as functions are pure, but it does not account for changes in things like data files. you can activate this using the `development_cache` method. The method takes a string name which identifies the cache, often you can just use `__name__`, unless the composer is in the `__main__` script. 128 | 129 | ```python 130 | cached_composer = composer.development_cache(__name__) 131 | ``` 132 | 133 | If something has changed that requires the cache to be invalidated you can use the `cache_invalidate` 134 | or `cache_clear` methods. `cache_invalidate` takes the names of teh functions you wish to invalidate, it will ensure any follow on functions are invalidated. `cache_clear` will clear the cache. 135 | 136 | ## Namespaces 137 | 138 | When logic gets complex, or similar logic needs to be reused in different spaces in one composer it can useful to use namespaces. Namespaces create a hierarchy of named scopes. that limits what functions arguments resolve to. Namespaces are separated with the double underscore (`__`). Namespaces are constructed using the `update_namespaces` method. For example: 139 | 140 | ```python 141 | from fn_graph import Composer 142 | 143 | def data(): 144 | return 5 145 | 146 | 147 | def b(data, factor): 148 | return data * factor 149 | 150 | 151 | def c(b): 152 | return b 153 | 154 | 155 | def combined_result(child_one__c, child_two__c): 156 | pass 157 | 158 | 159 | child = Composer().update(b, c) 160 | parent = ( 161 | Composer() 162 | .update_namespaces(child_one=child, child_two=child) 163 | .update(data, combined_result) 164 | .update_parameters(child_one__factor=3, child_two__factor=5) 165 | ) 166 | ``` 167 | 168 | Then the resulting graph would look like this: 169 | 170 | ![namespaces graphviz](namespaces.png) 171 | 172 | There are couple of things going on here. 173 | 174 | 1. You can see that function `c` resolves it's arguments within it;s namespace. 175 | 2. The `b` functions do not find a `data` function in their own namespace so they look to the parent namespace. 176 | 3. The combined_result specifically names it's argunents to pull from the namespaces using the double-underscore (`__`). 177 | 4. The parameters have been set differently in different namespaces (but could have been set the same by putting it in the top level namespace). 178 | 179 | These capabilities on their own allow you to construct (and reconstruct) very flexible logic. Sometimes though, given that arguments are resolved just by name it is useful to be able to create a link between one name and another. You can do this using the the `link` method. For example: 180 | 181 | ```python 182 | def calculated_factor(data): 183 | return data / 2 184 | 185 | 186 | factor_calc = Composer() 187 | factoring = Composer().update(calculated_factor) 188 | 189 | linked_parent = ( 190 | Composer() 191 | .update_namespaces(child_one=child, factoring=factoring) 192 | .update(data) 193 | .link(child_one__factor="factoring__calculated_factor") 194 | ) 195 | ``` 196 | 197 | Which looks like: 198 | 199 | ![linked namespaces graphviz](linked_namespaces.png) 200 | 201 | The link method gives you flexibility to link between different namespaces, and in general reconfigure your logic without having to write new functions. 202 | -------------------------------------------------------------------------------- /fn_graph/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from collections import defaultdict, namedtuple 4 | from functools import reduce 5 | from inspect import Parameter, signature 6 | import inspect 7 | from itertools import groupby 8 | from logging import getLogger 9 | from typing import Any, Callable, List 10 | 11 | import graphviz 12 | import networkx as nx 13 | from littleutils import strip_required_prefix, strip_required_suffix 14 | 15 | from fn_graph.calculation import NodeInstruction, get_execution_instructions 16 | from .caches import DevelopmentCache, SimpleCache, NullCache 17 | from .calculation import calculate 18 | 19 | log = getLogger(__name__) 20 | 21 | 22 | ComposerTestResult = namedtuple("TestResult", "name passed exception") 23 | """ 24 | The results of Composer.run_tests() 25 | """ 26 | 27 | # TODO cleverer use of caching - stop unnecessary reads 28 | 29 | 30 | class Composer: 31 | """ 32 | A function composer is responsible for orchestrating the composition 33 | and execution of graph of functions. 34 | 35 | Pure free functions are added to the composer, the names of the function 36 | arguments are used to determine how those functions are wired up into 37 | a directed acyclic graph. 38 | """ 39 | 40 | def __init__( 41 | self, 42 | *, 43 | _functions=None, 44 | _parameters=None, 45 | _cache=None, 46 | _tests=None, 47 | _source_map=None, 48 | ): 49 | # These are namespaced 50 | self._functions = _functions or {} 51 | self._tests = _tests or {} 52 | 53 | self._cache = _cache or NullCache() 54 | self._parameters = _parameters or {} 55 | self._source_map = _source_map or {} 56 | 57 | def _copy(self, **kwargs): 58 | return type(self)( 59 | **{ 60 | **dict( 61 | _functions=self._functions, 62 | _cache=self._cache, 63 | _parameters=self._parameters, 64 | _tests=self._tests, 65 | _source_map=self._source_map, 66 | ), 67 | **kwargs, 68 | } 69 | ) 70 | 71 | def update(self, *args: Callable, **kwargs: Callable) -> Composer: 72 | """ 73 | Add functions to the composer. 74 | 75 | Args: 76 | args: Positional arguments use the __name__ of the function as the reference 77 | in the graph. 78 | kwargs: Keyword arguments use the key as the name of the function in the graph. 79 | 80 | Returns: 81 | A new composer with the functions added. 82 | """ 83 | args_with_names = {arg.__name__: arg for arg in args} 84 | all_args = {**self._functions, **args_with_names, **kwargs} 85 | parameters = { 86 | k: v for k, v in self._parameters.items() if k not in args_with_names 87 | } 88 | for argname, argument in all_args.items(): 89 | if not callable(argument): 90 | raise Exception( 91 | f"Argument '{argname}' is not a function or callable. All arguments must be callable." 92 | ) 93 | 94 | return self._copy(_functions=all_args, _parameters=parameters) 95 | 96 | def update_without_prefix( 97 | self, prefix: str, *functions: Callable, **kwargs: Callable 98 | ) -> Composer: 99 | """ 100 | Given a prefix and a list of (named) functions, this adds the functions 101 | to the composer but first strips the prefix from their name. This is very 102 | useful to stop name shadowing. 103 | 104 | Args: 105 | prefix: The prefix to strip off the function names 106 | functions: functions to add while stripping the prefix 107 | kwargs: named functions to add 108 | Returns: 109 | A new composer with the functions added 110 | """ 111 | 112 | args_with_names = { 113 | strip_required_prefix(arg.__name__, prefix): arg for arg in functions 114 | } 115 | return self.update(**args_with_names, **kwargs) 116 | 117 | def update_without_suffix(self, suffix: str, *functions, **kwargs) -> Composer: 118 | """ 119 | Given a suffix and a list of (named) functions, this adds the functions 120 | to the composer but first strips the suffix from their name. This is very 121 | useful to stop name shadowing. 122 | 123 | Args: 124 | suffix: The suffix to strip off the function names 125 | functions: functions to add while stripping the suffix 126 | kwargs: named functions to add 127 | Returns: 128 | A new composer with the functions added 129 | """ 130 | 131 | args_with_names = { 132 | strip_required_suffix(arg.__name__, suffix): arg for arg in functions 133 | } 134 | return self.update(**args_with_names, **kwargs) 135 | 136 | def update_from(self, *composers: Composer) -> Composer: 137 | """ 138 | Create a new composer with all the functions from this composer 139 | as well as the the passed composers. 140 | 141 | Args: 142 | composers: The composers to take functions from 143 | 144 | Returns: 145 | A new Composer with all the input composers functions added. 146 | """ 147 | return reduce( 148 | lambda x, y: x.update(**y._functions) 149 | .update_parameters(**y._parameters) 150 | .update_tests(**y._tests), 151 | [self, *composers], 152 | ) 153 | 154 | def update_namespaces(self, **namespaces: Composer) -> Composer: 155 | """ 156 | Given a group of keyword named composers, create a series of functions 157 | namespaced by the keywords and drawn from the composers' functions. 158 | 159 | Args: 160 | namespaces: Composers that will be added at the namespace that corresponds \ 161 | to the arguments key 162 | 163 | Returns: 164 | A new Composer with all the input composers functions added as namespaces. 165 | """ 166 | return self._copy( 167 | **{ 168 | arg: { 169 | **getattr(self, arg), 170 | **{ 171 | "__".join([namespace, k]): value 172 | for namespace, composer in namespaces.items() 173 | for k, value in getattr(composer, arg).items() 174 | }, 175 | } 176 | for arg in ["_functions", "_parameters"] 177 | } 178 | ) 179 | 180 | def update_parameters(self, **parameters: Any) -> Composer: 181 | """ 182 | Allows you to pass static parameters to the graph, they will be exposed as callables. 183 | """ 184 | 185 | hydrated_parameters = {} 186 | for key, parameter in parameters.items(): 187 | if isinstance(parameter, tuple): 188 | hydrated_parameters[key] = parameter 189 | else: 190 | type_ = self._parameters.get(key, (type(parameter), None))[0] 191 | hydrated_parameters[key] = (type_, parameter) 192 | 193 | def serve_parameter(key, type_, value): 194 | def parameter(): 195 | cast_value = value 196 | if isinstance(cast_value, int) and issubclass(type_, float): 197 | cast_value = float(value) 198 | 199 | if not isinstance(cast_value, type_): 200 | raise Exception(f"Parameter '{key}' is not of type {type_}") 201 | return cast_value 202 | 203 | return parameter 204 | 205 | return self._copy( 206 | _parameters={**self._parameters, **hydrated_parameters}, 207 | _functions={ 208 | **self._functions, 209 | **{ 210 | # Have to capture the value eagerly 211 | key: serve_parameter(key, type_, value) 212 | for key, (type_, value) in hydrated_parameters.items() 213 | }, 214 | }, 215 | ) 216 | 217 | def update_tests(self, **tests) -> Composer: 218 | """ 219 | Adds tests to the composer. 220 | 221 | A test is a function that should check a property of the calculations 222 | results and raise an exception if they are not met. 223 | 224 | The work exactly the same way as functions in terms of resolution of 225 | arguments. They are run with the run_test method. 226 | """ 227 | return self._copy(_tests={**self._tests, **tests}) 228 | 229 | def link(self, **kwargs): 230 | """ 231 | Create a symlink between an argument name and a function output. 232 | This is a convenience method. For example: 233 | 234 | `f.link(my_unknown_argument="my_real_function")` 235 | 236 | is the same as 237 | 238 | `f.update(my_unknown_argument= lambda my_real_function: my_real_function)` 239 | 240 | """ 241 | 242 | def make_link_fn(source): 243 | fn = eval(f"lambda {source}: {source}") 244 | fn._is_fn_graph_link = True 245 | return fn 246 | 247 | fns = {key: make_link_fn(source) for key, source in kwargs.items()} 248 | return self.update(**fns) 249 | 250 | def functions(self): 251 | """ 252 | Dictionary of the functions 253 | """ 254 | return self._functions 255 | 256 | def parameters(self): 257 | """ 258 | Dictionary of the parameters of the form {key: (type, value)} 259 | """ 260 | return self._parameters 261 | 262 | def check(self, outputs=None): 263 | """ 264 | Returns a generator of errors if there are any errors in the function graph. 265 | """ 266 | if outputs is None: 267 | dag = self.dag() 268 | else: 269 | dag = self.ancestor_dag(outputs) 270 | 271 | cycles = list(nx.simple_cycles(dag)) 272 | if cycles: 273 | yield dict( 274 | type="cycle", 275 | message=f"Cycle found [{', '.join(cycles[0])}]. The function graph must be acyclic.", 276 | ) 277 | 278 | for unbound_fn, calling_fns in self.subgraph(dag.nodes())._unbound().items(): 279 | yield dict( 280 | type="unbound", 281 | message=f"Unbound function '{unbound_fn}' required.", 282 | function=unbound_fn, 283 | referers=calling_fns, 284 | ) 285 | 286 | def calculate( 287 | self, outputs, perform_checks=True, intermediates=False, progress_callback=None 288 | ): 289 | return calculate( 290 | self, outputs, perform_checks, intermediates, progress_callback 291 | ) 292 | 293 | def run_tests(self): 294 | """ 295 | Run all the composer tests. 296 | 297 | Returns: 298 | A generator of ComposerTestResults(name, passed, exception). 299 | """ 300 | 301 | all_referenced_functions = [ 302 | self._resolve_predecessor(tname, pname) 303 | for tname, fn in self._tests.items() 304 | for pname in signature(fn).parameters 305 | ] 306 | 307 | results = self.calculate(all_referenced_functions) 308 | 309 | for tname, fn in self._tests.items(): 310 | 311 | arguments = { 312 | pname: results[self._resolve_predecessor(tname, pname)] 313 | for pname in signature(fn).parameters 314 | } 315 | 316 | try: 317 | fn(**arguments) 318 | test_result = ComposerTestResult( 319 | name=tname, passed=True, exception=None 320 | ) 321 | except Exception as e: 322 | test_result = ComposerTestResult(name=tname, passed=False, exception=e) 323 | 324 | yield test_result 325 | 326 | def call(self, output): 327 | """ 328 | A convenience method to calculate a single output 329 | """ 330 | return self.calculate([output])[output] 331 | 332 | def precalculate(self, outputs): 333 | """ 334 | Create a new Composer where the results of the given functions have 335 | been pre-calculated. 336 | 337 | :param outputs: list of the names of the functions to pre-calculate 338 | :rtype: A composer 339 | """ 340 | results = self.calculate(outputs) 341 | return self.update( 342 | **{k: (lambda x: (lambda: x))(v) for k, v in results.items()} 343 | ) 344 | 345 | def __getattr__(self, name): 346 | """ 347 | Allow composed functions to be easily called. 348 | """ 349 | name = name.replace(".", "__") 350 | if name in self._functions: 351 | return lambda: self.calculate([name])[name] 352 | else: 353 | raise AttributeError( 354 | f"{self.__class__.__name__} object has no attribute '{name}', nor any composed function '{name}'." 355 | ) 356 | 357 | def raw_function(self, name): 358 | """ 359 | Access a raw function in the composer by name. Returns None if not found. 360 | """ 361 | return self._functions.get(name) 362 | 363 | def dag(self): 364 | """ 365 | Generates the DAG representing the function graph. 366 | 367 | :rtype: a networkx.DiGraph instance with function names as nodes 368 | """ 369 | 370 | G = nx.DiGraph() 371 | 372 | for key in self._functions: 373 | G.add_node(key) 374 | 375 | for key, fn in self._functions.items(): 376 | predecessors = self._resolve_predecessors(key) 377 | G.add_edges_from([(resolved, key) for _, resolved in predecessors]) 378 | 379 | return G 380 | 381 | def ancestor_dag(self, outputs): 382 | """ 383 | A dag of all the ancestors of the given outputs, i.e. the functions that must be calculated 384 | to for the given outputs. 385 | """ 386 | full_dag = self.dag() 387 | ancestors = set(outputs) | { 388 | pred for output in outputs for pred in nx.ancestors(full_dag, output) 389 | } 390 | return full_dag.subgraph(ancestors) 391 | 392 | def subgraph(self, function_names): 393 | """ 394 | Given a collection of function names this will create a new 395 | composer that only consists of those nodes. 396 | """ 397 | return self._copy( 398 | _functions={ 399 | k: self._functions[k] for k in function_names if k in self._functions 400 | }, 401 | _parameters={ 402 | k: v for k, v in self._parameters.items() if k in function_names 403 | }, 404 | ) 405 | 406 | def cache(self, backend=None) -> Composer: 407 | """ 408 | Create a new composer with a given cache backend. 409 | 410 | By default this is a SimpleCache. 411 | """ 412 | backend = backend or SimpleCache() 413 | return self._copy(_cache=backend) 414 | 415 | def development_cache(self, name, cache_dir=None) -> Composer: 416 | """ 417 | Create a new composer with a development cache setup 418 | """ 419 | return self.cache(DevelopmentCache(name, cache_dir)) 420 | 421 | def cache_clear(self): 422 | """ 423 | Clear the cache 424 | """ 425 | for key in self.dag(): 426 | self._cache.invalidate(self, key) 427 | 428 | def cache_invalidate(self, *nodes: List[str]): 429 | """ 430 | Invalidate the cache for all nodes affected by the 431 | given nodes (the descendants). 432 | """ 433 | to_invalidate = set() 434 | for node in nodes: 435 | to_invalidate.update(nx.descendants(self.dag(), node)) 436 | to_invalidate.add(node) 437 | 438 | for key in to_invalidate: 439 | self._cache.invalidate(self, key) 440 | 441 | def cache_graphviz(self, outputs=(), **kwargs): 442 | """ 443 | Display a graphviz with the cache invalidated nodes highlighted. 444 | """ 445 | instructions = get_execution_instructions(self, self.dag(), outputs) 446 | 447 | filter = self.ancestor_dag(outputs).nodes() if outputs else None 448 | 449 | def get_node_styles(instruction): 450 | return { 451 | NodeInstruction.IGNORE: dict(fillcolor="green"), 452 | NodeInstruction.RETRIEVE: dict(fillcolor="orange"), 453 | NodeInstruction.CALCULATE: dict(fillcolor="red"), 454 | }[instruction] 455 | 456 | # TODO: Be a bit more careful about people passing in conflicting params 457 | 458 | extra_node_styles = { 459 | node: get_node_styles(instruction) for node, instruction in instructions 460 | } 461 | 462 | return self.graphviz( 463 | extra_node_styles=extra_node_styles, filter=filter, **kwargs 464 | ) 465 | 466 | def graphviz( 467 | self, 468 | *, 469 | hide_parameters=False, 470 | expand_links=False, 471 | flatten=False, 472 | highlight=None, 473 | filter=None, 474 | extra_node_styles=None, 475 | ): 476 | """ 477 | Generates a graphviz.DiGraph that is suitable for display. 478 | 479 | This requires graphviz to be installed. 480 | 481 | The output can be directly viewed in a Jupyter notebook. 482 | """ 483 | extra_node_styles = extra_node_styles or {} 484 | highlight = highlight or [] 485 | dag = self.dag() 486 | 487 | if filter is None: 488 | filter = dag.nodes() 489 | 490 | unbound = set(self._unbound().keys()) 491 | 492 | # Recursively build from the tree 493 | def create_subgraph(tree, name=None): 494 | 495 | label = name 496 | if flatten: 497 | name = f"flat_{name}" if name else None 498 | else: 499 | name = f"cluster_{name}" if name else None 500 | 501 | g = graphviz.Digraph(name=name) 502 | if label: 503 | g.attr("graph", label=label, fontname="arial", title="") 504 | 505 | for k, v in tree.items(): 506 | if isinstance(v, str): 507 | name = v 508 | fn = self.raw_function(name) 509 | 510 | if hide_parameters and name in self._parameters: 511 | continue 512 | 513 | node_styles = dict( 514 | style="rounded, filled", fontname="arial", shape="rect" 515 | ) 516 | is_link = fn and getattr(fn, "_is_fn_graph_link", False) 517 | 518 | if name in highlight: 519 | color = "#7dc242" 520 | elif name in unbound: 521 | color = "red" 522 | elif name in self._parameters: 523 | color = "lightblue" 524 | 525 | else: 526 | color = "lightgrey" 527 | 528 | node_styles.update(dict(fillcolor=color)) 529 | 530 | if is_link and expand_links: 531 | node_styles.update(dict(fontcolor="darkgrey")) 532 | elif is_link: 533 | node_styles.update( 534 | dict(shape="circle", height="0.2", width="0.2") 535 | ) 536 | 537 | node_styles.update(extra_node_styles.get(v, {})) 538 | 539 | if is_link and not expand_links: 540 | label = "" 541 | elif flatten: 542 | label = name.replace("_", "\n") 543 | else: 544 | label = k.replace("_", "\n") 545 | 546 | if name in filter: 547 | g.node(name, label=label, **node_styles) 548 | else: 549 | g.subgraph(create_subgraph(v, k)) 550 | return g 551 | 552 | result = create_subgraph(self._build_name_tree()) 553 | result.attr("graph", rankdir="BT") 554 | for node in self.dag().nodes(): 555 | for _, pred in self._resolve_predecessors(node): 556 | if (not hide_parameters or pred not in self._parameters) and ( 557 | pred in filter and node in filter 558 | ): 559 | result.edge(pred, node) 560 | 561 | return result 562 | 563 | def set_source_map(self, source_map): 564 | """ 565 | Source maps allow you to override the code returned by get source. 566 | 567 | This is rarely used, and only in esoteric circumstances. 568 | """ 569 | return self._copy(_source_map=source_map) 570 | 571 | def get_source(self, key): 572 | """ 573 | Returns the source code that defines this function. 574 | """ 575 | fn = self.raw_function(key) 576 | is_parameter = key in self._parameters 577 | 578 | if key in self._source_map: 579 | return self._source_map[key] 580 | elif is_parameter: 581 | return f"{key} = lambda: {self._parameters[key]}" 582 | elif getattr(fn, "_is_fn_graph_link", False): 583 | parameter = list(inspect.signature(fn).parameters.keys())[0] 584 | return f"{key} = lambda {parameter}: {parameter}" 585 | else: 586 | try: 587 | return inspect.getsource(fn) 588 | except OSError: 589 | return "Source could not be located" 590 | 591 | def _build_name_tree(self): 592 | # Build up a tree of the subgraphs 593 | def recursive_tree(): 594 | return defaultdict(recursive_tree) 595 | 596 | tree = recursive_tree() 597 | unbound = set(self._unbound().keys()) 598 | 599 | for node in [*self.dag().nodes(), *unbound]: 600 | root = tree 601 | parts = node.split("__") 602 | for part in parts[:-1]: 603 | root = root[part] 604 | root[parts[-1]] = node 605 | return tree 606 | 607 | def _unbound(self): 608 | 609 | return { 610 | k: [t for _, t in v] 611 | for k, v in groupby( 612 | sorted( 613 | [ 614 | (arg, key) 615 | for key in self._functions 616 | for _, arg in self._resolve_predecessors(key) 617 | if arg not in self._functions 618 | ] 619 | ), 620 | key=lambda t: t[0], 621 | ) 622 | } 623 | 624 | def _resolve_predecessor(self, fname, pname): 625 | fparts = fname.split("__")[:] 626 | possible_preds = [ 627 | "__".join(fparts[:i] + [pname]) for i in range(0, len(fparts)) 628 | ] 629 | possible_preds.reverse() 630 | for possible in possible_preds: 631 | if possible in self._functions: 632 | return possible 633 | else: 634 | return possible_preds[0] 635 | 636 | def _resolve_var_predecessors(self, fname, pname): 637 | fparts = fname.split("__")[:] 638 | possible_preds = [ 639 | "__".join(fparts[:i] + [pname]) for i in range(0, len(fparts)) 640 | ] 641 | possible_preds.reverse() 642 | 643 | for possible_prefix in possible_preds: 644 | for function in self._functions.keys(): 645 | if function.startswith(possible_prefix): 646 | key = pname + function[len(possible_prefix) :] 647 | 648 | yield key, function 649 | 650 | def _resolve_predecessors(self, fname): 651 | 652 | if fname not in self._functions: 653 | return [] 654 | 655 | fn = self._functions[fname] 656 | sig = signature(fn) 657 | for key, parameter in sig.parameters.items(): 658 | if parameter.kind in ( 659 | parameter.KEYWORD_ONLY, 660 | parameter.POSITIONAL_OR_KEYWORD, 661 | parameter.POSITIONAL_ONLY, 662 | ): 663 | resolved_name = self._resolve_predecessor(fname, key) 664 | 665 | if parameter.default is not Parameter.empty: 666 | if resolved_name in self._functions: 667 | yield key, resolved_name 668 | else: 669 | yield key, resolved_name 670 | else: 671 | yield from self._resolve_var_predecessors(fname, key) 672 | -------------------------------------------------------------------------------- /fn_graph/caches.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import inspect 3 | import json 4 | import pickle 5 | from io import BytesIO 6 | from logging import getLogger 7 | from pathlib import Path 8 | 9 | 10 | log = getLogger(__name__) 11 | 12 | 13 | def fn_value(fn): 14 | if getattr(fn, "_is_fn_graph_link", False): 15 | return ",".join(inspect.signature(fn).parameters.keys()) 16 | else: 17 | return inspect.getsource(fn) 18 | 19 | 20 | def hash_fn(composer, key, use_id=False): 21 | value = ( 22 | composer._parameters[key][1] 23 | if key in composer._parameters 24 | else composer._functions[key] 25 | ) 26 | 27 | if use_id: 28 | log.debug(f"Fn Value on %s for value %s is ID", key, id(value)) 29 | return id(value) 30 | 31 | if callable(value): 32 | log.debug("Fn Value on %s for value %r is callable", key, value) 33 | buffer = fn_value(value).encode("utf-8") 34 | return hashlib.sha256(buffer).digest() 35 | else: 36 | log.debug("Fn Value on %s for value %r is not callable", key, value) 37 | buffer = BytesIO() 38 | pickle.dump(value, buffer) 39 | return hashlib.sha256(buffer.getvalue()).digest() 40 | 41 | 42 | class NullCache: 43 | """ 44 | Performs no caching. 45 | 46 | Used as a base class for other caches to indicate that they all have the same signature. 47 | """ 48 | 49 | def valid(self, composer, key): 50 | return False 51 | 52 | def get(self, composer, key): 53 | pass 54 | 55 | def set(self, composer, key, value): 56 | pass 57 | 58 | def invalidate(self, composer, key): 59 | pass 60 | 61 | 62 | class SimpleCache(NullCache): 63 | """ 64 | Stores results in memory, performs no automatic invalidation. 65 | 66 | Set hash_parameters=False to only check the object identities of parameters 67 | when checking cache validity. This is slightly less robust to buggy usage 68 | but is much faster for big parameter objects. 69 | 70 | DO NOT USE THIS IN DEVELOPMENT OR A NOTEBOOK! 71 | """ 72 | 73 | def __init__(self, hash_parameters=False): 74 | self.cache = {} 75 | self.hashes = {} 76 | self.hash_parameters = hash_parameters 77 | 78 | def _hash(self, composer, key): 79 | return hash_fn(composer, key, use_id=not self.hash_parameters) 80 | 81 | def valid(self, composer, key): 82 | log.debug("Length %s", len(composer._functions)) 83 | if key in self.hashes: 84 | current_hash = self._hash(composer, key) 85 | stored_hash = self.hashes[key] 86 | matches = current_hash == stored_hash 87 | log.debug(f"hash test {key} {matches}: {current_hash} {stored_hash}") 88 | return matches 89 | else: 90 | log.debug(f"cache test {key} {key in self.cache}") 91 | return key in self.cache 92 | 93 | def get(self, composer, key): 94 | return self.cache[key] 95 | 96 | def set(self, composer, key, value): 97 | self.cache[key] = value 98 | if key in composer.parameters(): 99 | self.hashes[key] = self._hash(composer, key) 100 | 101 | def invalidate(self, composer, key): 102 | 103 | if key in self.cache: 104 | del self.cache[key] 105 | 106 | if key in self.hashes: 107 | del self.hashes[key] 108 | 109 | 110 | class DevelopmentCache(NullCache): 111 | """ 112 | Store cache on disk, analyses the coe for changes and performs automatic 113 | invalidation. 114 | 115 | This is only for use during development! DO NOT USE THIS IN PRODUCTION! 116 | 117 | The analysis of code changes is limited, it assumes that all functions are 118 | pure, and tht there have been no important changes in the outside environment, 119 | like a file that has been changed, 120 | """ 121 | 122 | def __init__(self, name, cache_dir): 123 | self.name = name 124 | 125 | if cache_dir is None: 126 | cache_dir = ".fn_graph_cache" 127 | 128 | self.cache_dir = Path(cache_dir) 129 | self.cache_root.mkdir(parents=True, exist_ok=True) 130 | 131 | @property 132 | def cache_root(self): 133 | return self.cache_dir / self.name 134 | 135 | def valid(self, composer, key): 136 | self.cache_root.mkdir(parents=True, exist_ok=True) 137 | pickle_file_path = self.cache_root / f"{key}.data" 138 | info_file_path = self.cache_root / f"{key}.info.json" 139 | fn_hash_path = self.cache_root / f"{key}.fn.hash" 140 | 141 | exists = pickle_file_path.exists() and info_file_path.exists() 142 | 143 | log.debug( 144 | "Checking development cache '%s' for key '%s': exists = %s", 145 | self.name, 146 | key, 147 | exists, 148 | ) 149 | 150 | if not exists: 151 | return False 152 | 153 | current_hash = hash_fn(composer, key, False) 154 | with open(fn_hash_path, "rb") as f: 155 | previous_hash = f.read() 156 | 157 | if current_hash != previous_hash: 158 | 159 | log.debug( 160 | "Hash difference in cache '%s' for key '%s' is current %r vs previous %r", 161 | self.name, 162 | key, 163 | current_hash, 164 | previous_hash, 165 | ) 166 | 167 | return False 168 | 169 | log.debug("Valid development cache '%s' for key '%s'", self.name, key) 170 | 171 | return True 172 | 173 | def get(self, composer, key): 174 | 175 | log.debug("Retrieving from development cache '%s' for key '%s'", self.name, key) 176 | self.cache_root.mkdir(parents=True, exist_ok=True) 177 | pickle_file_path = self.cache_root / f"{key}.data" 178 | info_file_path = self.cache_root / f"{key}.info.json" 179 | 180 | with open(info_file_path, "r") as f: 181 | information = json.load(f) 182 | 183 | format = information["format"] 184 | 185 | with open(pickle_file_path, "rb") as f: 186 | if format == "pickle": 187 | return pickle.load(f) 188 | elif format == "pandas-parquet": 189 | import pandas as pd 190 | 191 | return pd.read_parquet(f) 192 | else: 193 | raise Exception(f"Unknown caching fn_graph format: {format}") 194 | 195 | def set(self, composer, key, value): 196 | 197 | log.debug("Writing to development cache '%s' for key '%s'", self.name, key) 198 | self.cache_root.mkdir(parents=True, exist_ok=True) 199 | pickle_file_path = self.cache_root / f"{key}.data" 200 | info_file_path = self.cache_root / f"{key}.info.json" 201 | fn_hash_path = self.cache_root / f"{key}.fn.hash" 202 | 203 | # This is a low-fi way to checK the type without adding requirements 204 | # I am concerned it is fragile 205 | if str(type(value)) == "": 206 | format = "pandas-parquet" 207 | else: 208 | format = "pickle" 209 | 210 | saved = False 211 | if format == "pandas-parquet": 212 | try: 213 | with open(pickle_file_path, "wb") as f: 214 | value.to_parquet(f) 215 | saved = True 216 | except: 217 | saved = False 218 | 219 | if not saved: 220 | format = "pickle" 221 | with open(pickle_file_path, "wb") as f: 222 | pickle.dump(value, f) 223 | 224 | with open(fn_hash_path, "wb") as f: 225 | f.write(hash_fn(composer, key)) 226 | 227 | with open(info_file_path, "w") as f: 228 | json.dump({"format": format}, f) 229 | 230 | def invalidate(self, composer, key): 231 | log.debug("Invalidation in development cache '%s' for key '%s'", self.name, key) 232 | self.cache_root.mkdir(parents=True, exist_ok=True) 233 | paths = [ 234 | self.cache_root / f"{key}.data", 235 | self.cache_root / f"{key}.fn.hash", 236 | self.cache_root / f"{key}.info.json", 237 | ] 238 | 239 | for path in paths: 240 | if path.exists(): 241 | path.unlink() 242 | -------------------------------------------------------------------------------- /fn_graph/calculation.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import traceback 3 | from collections import Counter 4 | from enum import Enum 5 | from functools import reduce 6 | from inspect import Parameter, signature 7 | from logging import getLogger 8 | 9 | import networkx as nx 10 | from littleutils import ensure_list_if_string 11 | 12 | log = getLogger(__name__) 13 | 14 | 15 | class NodeInstruction(Enum): 16 | CALCULATE = 1 17 | RETRIEVE = 2 18 | IGNORE = 3 19 | 20 | 21 | def get_execution_instructions(composer, dag, outputs): 22 | 23 | direct_invalid_nodes = { 24 | node for node in dag if not composer._cache.valid(composer, node) 25 | } 26 | 27 | invalid_nodes = ( 28 | reduce( 29 | lambda a, b: a | b, 30 | [nx.descendants(dag, node) for node in direct_invalid_nodes] + [set()], 31 | ) 32 | | direct_invalid_nodes 33 | ) 34 | 35 | must_be_retrieved = ( 36 | { 37 | node 38 | for node in dag.nodes() 39 | if any(succ in invalid_nodes for succ in dag.successors(node)) 40 | } 41 | | set(outputs) 42 | ) - invalid_nodes 43 | 44 | log.debug("Invalid nodes %s", invalid_nodes) 45 | log.debug("Retrieved Nodes %s", must_be_retrieved) 46 | 47 | execution_order = list(nx.topological_sort(dag)) 48 | execution_instructions = [] 49 | for node in execution_order: 50 | log.debug(node) 51 | if node in invalid_nodes: 52 | node_instruction = NodeInstruction.CALCULATE 53 | elif node in must_be_retrieved: 54 | node_instruction = NodeInstruction.RETRIEVE 55 | else: 56 | node_instruction = NodeInstruction.IGNORE 57 | execution_instructions.append((node, node_instruction)) 58 | 59 | return execution_instructions 60 | 61 | 62 | def maintain_cache_consistency(composer): 63 | dag = composer.dag() 64 | 65 | direct_invalid_nodes = { 66 | node 67 | for node in composer._functions 68 | if node in dag and not composer._cache.valid(composer, node) 69 | } 70 | log.debug("Direct invalid nodes %s", direct_invalid_nodes) 71 | 72 | # If a node is invalidate all it's descendents must be made invalid 73 | indirect_invalid_nodes = ( 74 | reduce( 75 | lambda a, b: a | b, 76 | [nx.descendants(dag, node) for node in direct_invalid_nodes] + [set()], 77 | ) 78 | ) - direct_invalid_nodes 79 | log.debug("Indirect invalid nodes %s", indirect_invalid_nodes) 80 | 81 | for node in indirect_invalid_nodes: 82 | composer._cache.invalidate(composer, node) 83 | 84 | 85 | def coalesce_argument_names(function, predecessor_results): 86 | sig = signature(function) 87 | 88 | positional_names = [] 89 | args_name = None 90 | keyword_names = [] 91 | kwargs_name = None 92 | 93 | for key, parameter in sig.parameters.items(): 94 | if parameter.kind in ( 95 | parameter.KEYWORD_ONLY, 96 | parameter.POSITIONAL_OR_KEYWORD, 97 | parameter.POSITIONAL_ONLY, 98 | ): 99 | if ( 100 | parameter.default is not Parameter.empty 101 | and key not in predecessor_results 102 | ): 103 | continue 104 | 105 | if args_name is None: 106 | positional_names.append(key) 107 | else: 108 | keyword_names.append(key) 109 | elif parameter.kind == parameter.VAR_POSITIONAL: 110 | args_name = key 111 | elif parameter.kind == parameter.VAR_KEYWORD: 112 | kwargs_name = key 113 | 114 | return positional_names, args_name, keyword_names, kwargs_name 115 | 116 | 117 | def coalesce_arguments(function, predecessor_results): 118 | positional_names, args_name, keyword_names, kwargs_name = coalesce_argument_names( 119 | function, predecessor_results 120 | ) 121 | 122 | positional = [predecessor_results.pop(name) for name in positional_names] 123 | args = [ 124 | predecessor_results.pop(name) 125 | for name in list(predecessor_results.keys()) 126 | if name.startswith(args_name) 127 | ] 128 | keywords = {name: predecessor_results.pop(name) for name in keyword_names} 129 | kwargs = { 130 | name: predecessor_results.pop(name) 131 | for name in list(predecessor_results) 132 | if name.startswith(kwargs_name) 133 | } 134 | 135 | assert len(predecessor_results) == 0 136 | 137 | return positional, args, keywords, kwargs 138 | 139 | 140 | def calculate_collect_exceptions( 141 | composer, 142 | outputs, 143 | perform_checks=True, 144 | intermediates=False, 145 | progress_callback=None, 146 | raise_immediately=False, 147 | ): 148 | """ 149 | Executes the required parts of the function graph to product results 150 | for the given outputs. 151 | 152 | Args: 153 | composer: The composer to calculate 154 | outputs: list of the names of the functions to calculate 155 | perform_checks: if true error checks are performed before calculation 156 | intermediates: if true the results of all functions calculated will be returned 157 | progress_callback: a callback that is called as the calculation progresses,\ 158 | this be of the form `callback(event_type, details)` 159 | 160 | Returns: 161 | Tuple: Tuple of (results, exception_info), results is a dictionary of results keyed by 162 | function name, exception_info (etype, evalue, etraceback, node) is the information 163 | about the exception if there was any. 164 | """ 165 | outputs = ensure_list_if_string(outputs) 166 | 167 | progress_callback = progress_callback or (lambda *args, **kwargs: None) 168 | 169 | progress_callback("start_calculation", dict(outputs=outputs)) 170 | 171 | if perform_checks: 172 | try: 173 | for name in outputs: 174 | if name not in composer._functions: 175 | raise Exception( 176 | f"'{name}' is not a composed function in this {composer.__class__.__name__} object." 177 | ) 178 | 179 | for error in composer.check(outputs): 180 | raise Exception(error) 181 | except: 182 | if raise_immediately: 183 | raise 184 | else: 185 | etype, evalue, etraceback = sys.exc_info() 186 | 187 | return {}, (etype, evalue, etraceback, None) 188 | 189 | maintain_cache_consistency(composer) 190 | 191 | # Limit to only the functions we care about 192 | dag = composer.ancestor_dag(outputs) 193 | 194 | if intermediates: 195 | outputs = dag.nodes() 196 | 197 | execution_instructions = get_execution_instructions(composer, dag, outputs) 198 | log.debug(execution_instructions) 199 | 200 | # Results store 201 | results = {} 202 | 203 | # Number of time a functions results still needs to be accessed 204 | remaining_usage_counts = Counter(pred for pred, _ in dag.edges()) 205 | progress_callback( 206 | "prepared_calculation", 207 | dict(execution_instructions=execution_instructions, execution_graph=dag), 208 | ) 209 | log.debug("Starting execution") 210 | for node, instruction in execution_instructions: 211 | progress_callback( 212 | "start_step", dict(name=node, execution_instruction=instruction) 213 | ) 214 | try: 215 | predecessors = list(composer._resolve_predecessors(node)) 216 | 217 | if instruction == NodeInstruction.IGNORE: 218 | log.debug("Ignoring function '%s'", node) 219 | 220 | elif instruction == NodeInstruction.RETRIEVE: 221 | log.debug("Retrieving function '%s'", node) 222 | try: 223 | progress_callback("start_cache_retrieval", dict(name=node)) 224 | results[node] = composer._cache.get(composer, node) 225 | finally: 226 | progress_callback("end_cache_retrieval", dict(name=node)) 227 | else: 228 | log.debug("Calculating function '%s'", node) 229 | function = composer._functions[node] 230 | 231 | # Pull up arguments 232 | predecessor_results = { 233 | parameter: results[pred] for parameter, pred in predecessors 234 | } 235 | positional, args, keywords, kwargs = coalesce_arguments( 236 | function, predecessor_results 237 | ) 238 | 239 | try: 240 | progress_callback("start_function", dict(name=node)) 241 | result = function(*positional, *args, **keywords, **kwargs) 242 | except Exception as e: 243 | if raise_immediately: 244 | raise 245 | else: 246 | etype, evalue, etraceback = sys.exc_info() 247 | 248 | return results, (etype, evalue, etraceback, node) 249 | finally: 250 | progress_callback("end_function", dict(name=node)) 251 | 252 | results[node] = result 253 | 254 | try: 255 | progress_callback("start_cache_store", dict(name=node)) 256 | composer._cache.set(composer, node, result) 257 | finally: 258 | progress_callback("end_cache_store", dict(name=node)) 259 | 260 | # Eject results from memory once the are not needed 261 | remaining_usage_counts.subtract([pred for _, pred in predecessors]) 262 | ready_to_eject = [ 263 | key 264 | for key, value in remaining_usage_counts.items() 265 | if value == 0 and key not in outputs 266 | ] 267 | for key in ready_to_eject: 268 | assert key not in outputs 269 | remaining_usage_counts.pop(key) 270 | results.pop(key, "not_found") 271 | finally: 272 | progress_callback( 273 | "end_step", 274 | dict( 275 | name=node, 276 | execution_instruction=instruction, 277 | result=results.get(node), 278 | ), 279 | ) 280 | 281 | # We should just be left with the results 282 | return results, None 283 | 284 | 285 | def calculate(*args, **kwargs): 286 | """ 287 | Executes the required parts of the function graph to product results 288 | for the given outputs. 289 | 290 | Args: 291 | composer: The composer to calculate 292 | outputs: list of the names of the functions to calculate 293 | perform_checks: if true error checks are performed before calculation 294 | intermediates: if true the results of all functions calculated will be returned 295 | progress_callback: a callback that is called as the calculation progresses,\ 296 | this be of the form `callback(event_type, details)` 297 | 298 | Returns: 299 | Dictionary: Dictionary of results keyed by function name 300 | """ 301 | results, _ = calculate_collect_exceptions(*args, raise_immediately=True, **kwargs) 302 | return results 303 | -------------------------------------------------------------------------------- /fn_graph/examples/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BusinessOptics/fn_graph/50ef05ce07c069383850cbdba190af790278419b/fn_graph/examples/__init__.py -------------------------------------------------------------------------------- /fn_graph/examples/broken.py: -------------------------------------------------------------------------------- 1 | "An example with a broken composer." 2 | 3 | #%% 4 | from pathlib import Path 5 | from random import choice, random 6 | 7 | import pandas as pd 8 | import numpy as np 9 | import plotly.express as px 10 | import math 11 | 12 | from fn_graph import Composer 13 | 14 | prices = [random() * 100_000 + 50000 for _ in range(10)] 15 | 16 | f = ( 17 | Composer() 18 | .update( 19 | wave=lambda frequency, amplitude: pd.DataFrame(dict(x=range(501))).assign( 20 | y=lambda df: amplitude 21 | * np.cos(df.x / 500 * math.pi * 3) 22 | * np.sin(df.x / 500 * math.pi * frequency) 23 | ), 24 | plot=lambda wave: px.line(wave, x="x", y="y"), 25 | broken=lambda wave, missing: wave, 26 | ) 27 | .update_parameters(frequency=(float, 1), amplitude=(float, 1)) 28 | ) 29 | 30 | # %% 31 | -------------------------------------------------------------------------------- /fn_graph/examples/caching.py: -------------------------------------------------------------------------------- 1 | "An example showing caching behaviour." 2 | 3 | #%% 4 | from pathlib import Path 5 | from random import choice, random 6 | 7 | 8 | import pandas as pd 9 | import numpy as np 10 | import plotly.express as px 11 | import math 12 | 13 | from fn_graph import Composer 14 | 15 | prices = [random() * 100_000 + 50000 for _ in range(10)] 16 | 17 | f = ( 18 | Composer() 19 | .update( 20 | wave=lambda frequency, amplitude: pd.DataFrame(dict(x=range(501))).assign( 21 | y=lambda df: amplitude 22 | * np.cos(df.x / 500 * math.pi * 3) 23 | * np.sin(df.x / 500 * math.pi * frequency) 24 | ), 25 | plot=lambda wave: px.line(wave, x="x", y="y"), 26 | ) 27 | .update_parameters(frequency=(float, 1), amplitude=(float, 1)) 28 | ).cache() 29 | 30 | 31 | # %% 32 | -------------------------------------------------------------------------------- /fn_graph/examples/car_savings.py: -------------------------------------------------------------------------------- 1 | """ 2 | A simple example showing basic functionality. 3 | """ 4 | #%% 5 | from random import choice, random 6 | 7 | import pandas as pd 8 | import plotly.express as px 9 | from fn_graph import Composer 10 | 11 | prices = [random() * 100_000 + 50000 for _ in range(10)] 12 | 13 | 14 | def get_car_prices(): 15 | df = pd.DataFrame( 16 | dict( 17 | model=[choice(["corolla", "beetle", "ferrari"]) for _ in range(10)], 18 | price=prices, 19 | ) 20 | ) 21 | 22 | return df 23 | 24 | 25 | def get_mean_car_price(car_prices, season="summer"): 26 | if season != "summer": 27 | return car_prices.price.mean() / 2 28 | else: 29 | return car_prices.price.mean() 30 | 31 | 32 | def get_cheaper_cars(car_prices, your_car_price): 33 | df = car_prices 34 | return df[df.price < your_car_price] 35 | 36 | 37 | def get_savings_on_cheaper_cars(cheaper_cars, mean_car_price): 38 | return cheaper_cars.assign(savings=lambda df: mean_car_price - df.price) 39 | 40 | 41 | def get_burger_savings(savings_on_cheaper_cars, price_of_a_burger): 42 | return savings_on_cheaper_cars.assign( 43 | burgers_saved=lambda df: df.savings / price_of_a_burger 44 | ) 45 | 46 | 47 | def get_savings_histogram(burger_savings): 48 | return px.histogram(burger_savings, x="burgers_saved") 49 | 50 | 51 | f = ( 52 | Composer() 53 | .update_without_prefix( 54 | "get_", 55 | get_car_prices, 56 | get_cheaper_cars, 57 | get_mean_car_price, 58 | get_savings_on_cheaper_cars, 59 | get_burger_savings, 60 | get_savings_histogram, 61 | ) 62 | .update_parameters(your_car_price=(int, 100_000), price_of_a_burger=(float, 100)) 63 | ) 64 | -------------------------------------------------------------------------------- /fn_graph/examples/credit.py: -------------------------------------------------------------------------------- 1 | """ 2 | A credit model that uses a machine learning model to estimate the expected 3 | value of the the remaining loans in a loan book. 4 | 5 | The example shows how to integrate a machine learning model with some simple 6 | domain information to get contextualized result. It also show cases how statistical 7 | libraries like Seaborn can be used to investigate data before hand. 8 | """ 9 | 10 | from pathlib import Path 11 | 12 | import pandas as pd 13 | import seaborn as sns 14 | from sklearn.ensemble import RandomForestClassifier 15 | 16 | from fn_graph import Composer 17 | 18 | data_path = Path(__file__).parent 19 | 20 | 21 | def loan_data(): 22 | """ 23 | Load the loan data 24 | """ 25 | return pd.read_csv(data_path / "credit_data.csv") 26 | 27 | 28 | def training_data(loan_data): 29 | """ 30 | Ignore currently live loans 31 | """ 32 | return loan_data[loan_data.status != "LIVE"] 33 | 34 | 35 | def investigate_data(training_data): 36 | """ 37 | Use a seaborn pairplot to get a feel for the data 38 | """ 39 | return sns.pairplot(training_data.sample(100), hue="status") 40 | 41 | 42 | def training_features(training_data: pd.DataFrame): 43 | """ 44 | One hot encode gender and dro =p columns not used in training 45 | """ 46 | return pd.get_dummies( 47 | training_data.drop(columns=["outstanding_balance", "status", "account_no"]) 48 | ) 49 | 50 | 51 | def training_target(training_data): 52 | """ 53 | Convert the target variable to a boolean 54 | """ 55 | return training_data.status == "DEFAULT" 56 | 57 | 58 | def model(training_features, training_target): 59 | """ 60 | Fit a model 61 | """ 62 | model = RandomForestClassifier() 63 | model.fit(training_features, training_target) 64 | return model 65 | 66 | 67 | def prediction_data(loan_data): 68 | """ 69 | Only consider the currently live loans 70 | """ 71 | return loan_data[loan_data.status == "LIVE"] 72 | 73 | 74 | def prediction_features(prediction_data: pd.DataFrame): 75 | """ 76 | Prepare the prediction features 77 | """ 78 | return pd.get_dummies( 79 | prediction_data.drop(columns=["outstanding_balance", "status", "account_no"]) 80 | ) 81 | 82 | 83 | def probability_of_default(model, prediction_features): 84 | """ 85 | Predict the probability of default using the trained model 86 | """ 87 | return model.predict_proba(prediction_features)[:, 1] 88 | 89 | 90 | def expected_outstanding_repayment(prediction_data, probability_of_default): 91 | """ 92 | Calculate the expected repayment for each loan 93 | """ 94 | return prediction_data.assign(probability_of_default=probability_of_default).assign( 95 | expected_repayment=lambda df: df.outstanding_balance 96 | * (1 - df.probability_of_default) 97 | ) 98 | 99 | 100 | def value_of_live_book(expected_outstanding_repayment): 101 | """ 102 | The total remianing value of the loan book 103 | """ 104 | return expected_outstanding_repayment.expected_repayment.sum() 105 | 106 | 107 | composer = Composer().update( 108 | loan_data, 109 | training_data, 110 | training_features, 111 | training_target, 112 | investigate_data, 113 | model, 114 | prediction_data, 115 | prediction_features, 116 | probability_of_default, 117 | expected_outstanding_repayment, 118 | value_of_live_book, 119 | ) 120 | 121 | # Just for uniformity with the rest of the examples 122 | f = composer 123 | -------------------------------------------------------------------------------- /fn_graph/examples/finance.py: -------------------------------------------------------------------------------- 1 | """ 2 | In finance, the Sharpe ratio (also known as the Sharpe index, the Sharpe measure, 3 | and the reward-to-variability ratio) measures the performance of an investment 4 | (e.g., a security or portfolio) compared to a risk-free asset, after adjusting 5 | for its risk. It is defined as the difference between the returns of the 6 | investment and the risk-free return, divided by the standard deviation of the 7 | investment (i.e., its volatility). It represents the additional amount of return 8 | that an investor receives per unit of increase in risk. 9 | 10 | This shows how to calculate a the Sharoe ratio for a small portfolio of shares. The 11 | share data us pulled from yahoo finance and the analysis is done in pandas. We assume 12 | a risk free rate of zero. 13 | """ 14 | 15 | from datetime import date 16 | from math import sqrt 17 | from pathlib import Path 18 | 19 | import matplotlib.pyplot as plt 20 | import pandas as pd 21 | import yfinance as yf 22 | from fn_graph import Composer 23 | from pandas.plotting import register_matplotlib_converters 24 | 25 | register_matplotlib_converters() 26 | plt.style.use("fivethirtyeight") 27 | 28 | 29 | def closing_prices(share_allocations, start_date, end_date): 30 | """ 31 | The closing prices of our portfolio pulled with yfinance. 32 | """ 33 | data = yf.download( 34 | " ".join(share_allocations.keys()), start=start_date, end=end_date 35 | ) 36 | return data["Close"] 37 | 38 | 39 | def normalised_returns(closing_prices): 40 | """ 41 | Normalise the returns as a ratio of the initial price. 42 | """ 43 | return closing_prices / closing_prices.iloc[0, :] 44 | 45 | 46 | def positions(normalised_returns, share_allocations, initial_total_position): 47 | """ 48 | Our total positions ovr time given an initial allocation. 49 | """ 50 | 51 | allocations = pd.DataFrame( 52 | { 53 | symbol: normalised_returns[symbol] * allocation 54 | for symbol, allocation in share_allocations.items() 55 | } 56 | ) 57 | 58 | return allocations * initial_total_position 59 | 60 | 61 | def total_position(positions): 62 | """ 63 | The total value of out portfolio 64 | """ 65 | return positions.sum(axis=1) 66 | 67 | 68 | def positions_plot(positions): 69 | return positions.plot.line(figsize=(10, 8)) 70 | 71 | 72 | def cumulative_return(total_position): 73 | """ 74 | The cumulative return of our portfolio 75 | """ 76 | return 100 * (total_position[-1] / total_position[0] - 1) 77 | 78 | 79 | def daily_return(total_position): 80 | """ 81 | The daily return of our portfolio 82 | """ 83 | return total_position.pct_change(1) 84 | 85 | 86 | def sharpe_ratio(daily_return): 87 | """ 88 | The sharpe ratio of the portfolio assuming a zero risk free rate. 89 | """ 90 | return daily_return.mean() / daily_return.std() 91 | 92 | 93 | def annual_sharpe_ratio(sharpe_ratio, trading_days_in_a_year=252): 94 | return sharpe_ratio * sqrt(trading_days_in_a_year) 95 | 96 | 97 | composer = ( 98 | Composer() 99 | .update( 100 | closing_prices, 101 | normalised_returns, 102 | positions, 103 | total_position, 104 | positions_plot, 105 | cumulative_return, 106 | daily_return, 107 | sharpe_ratio, 108 | annual_sharpe_ratio, 109 | ) 110 | .update_parameters( 111 | share_allocations={"AAPL": 0.25, "MSFT": 0.25, "ORCL": 0.30, "IBM": 0.2}, 112 | initial_total_position=100_000_000, 113 | start_date="2017-01-01", 114 | end_date=date.today(), 115 | ) 116 | .cache() 117 | ) 118 | 119 | # Just for uniformity with the rest of the examples 120 | f = composer 121 | -------------------------------------------------------------------------------- /fn_graph/examples/machine_learning.py: -------------------------------------------------------------------------------- 1 | """ 2 | A simple machine learning example that builds a classifier for the standard iris dataset. 3 | 4 | This example uses scikit-learn to build a a classifier to detect the species of an iris 5 | flower based on attributes of the flower. Based on the parameters different types of models 6 | can be trained, and preprocessing can be turned on and off. It also show cases the integration 7 | of visualisations to measure the accuracy of the model. 8 | """ 9 | 10 | from fn_graph import Composer 11 | import sklearn, sklearn.datasets, sklearn.svm, sklearn.linear_model, sklearn.metrics 12 | from sklearn.model_selection import train_test_split 13 | import pandas as pd 14 | import numpy as np 15 | import seaborn as sns 16 | import matplotlib.pylab as plt 17 | 18 | 19 | def iris(): 20 | """ 21 | Load the classic iris dataset 22 | """ 23 | return sklearn.datasets.load_iris() 24 | 25 | 26 | def data(iris): 27 | """ 28 | Pull out the data as pandas DataFrame 29 | """ 30 | df_train = pd.DataFrame( 31 | iris.data, columns=["feature{}".format(i) for i in range(4)] 32 | ) 33 | return df_train.assign(y=iris.target) 34 | 35 | 36 | def investigate_data(data): 37 | """ 38 | Check for any visual correlations using seaborn 39 | """ 40 | return sns.pairplot(data, hue="y") 41 | 42 | 43 | def preprocess_data(data, do_preprocess): 44 | """ 45 | Preprocess the data by scaling depending on the parameter 46 | 47 | We make sure we don't mutate the data because that is better practice. 48 | """ 49 | processed = data.copy() 50 | if do_preprocess: 51 | processed.iloc[:, :-1] = sklearn.preprocessing.scale(processed.iloc[:, :-1]) 52 | return processed 53 | 54 | 55 | def split_data(preprocess_data): 56 | """ 57 | Split the data into test and train sets 58 | """ 59 | return dict( 60 | zip( 61 | ("training_features", "test_features", "training_target", "test_target"), 62 | train_test_split(preprocess_data.iloc[:, :-1], preprocess_data["y"]), 63 | ) 64 | ) 65 | 66 | 67 | # This is done verbosely purpose, but it could be more concise 68 | 69 | 70 | def training_features(split_data): 71 | return split_data["training_features"] 72 | 73 | 74 | def training_target(split_data): 75 | return split_data["training_target"] 76 | 77 | 78 | def test_features(split_data): 79 | return split_data["test_features"] 80 | 81 | 82 | def test_target(split_data): 83 | return split_data["test_target"] 84 | 85 | 86 | def model(training_features, training_target, model_type): 87 | """ 88 | Train the model 89 | """ 90 | if model_type == "ols": 91 | model = sklearn.linear_model.LogisticRegression() 92 | elif model_type == "svm": 93 | model = sklearn.svm.SVC() 94 | else: 95 | raise ValueError("invalid model selection, choose either 'ols' or 'svm'") 96 | model.fit(training_features, training_target) 97 | return model 98 | 99 | 100 | def predictions(model, test_features): 101 | """ 102 | Make some predictions foo the test data 103 | """ 104 | return model.predict(test_features) 105 | 106 | 107 | def classification_metrics(predictions, test_target): 108 | """ 109 | Show some standard classification metrics 110 | """ 111 | return sklearn.metrics.classification_report(test_target, predictions) 112 | 113 | 114 | def plot_confusion_matrix( 115 | cm, target_names, title="Confusion matrix", cmap=plt.cm.Blues 116 | ): 117 | """ 118 | Plots a confusion matrix using matplotlib. 119 | 120 | This is just a regular function that is not in the composer. 121 | Shamelessly taken from https://scikit-learn.org/0.15/auto_examples/model_selection/plot_confusion_matrix.html 122 | """ 123 | 124 | plt.imshow(cm, interpolation="nearest", cmap=cmap) 125 | plt.title(title) 126 | plt.colorbar() 127 | tick_marks = np.arange(len(target_names)) 128 | plt.xticks(tick_marks, target_names, rotation=45) 129 | plt.yticks(tick_marks, target_names) 130 | plt.tight_layout() 131 | plt.ylabel("True label") 132 | plt.xlabel("Predicted label") 133 | return plt.gcf() 134 | 135 | 136 | def confusion_matrix(predictions, test_target): 137 | """ 138 | Show the confusion matrix 139 | """ 140 | cm = sklearn.metrics.confusion_matrix(test_target, predictions) 141 | return plot_confusion_matrix(cm, ["setosa", "versicolor", "virginica"]) 142 | 143 | 144 | f = ( 145 | Composer() 146 | .update_parameters( 147 | # Parameter controlling the model type (ols, svc) 148 | model_type="ols", 149 | # Parameter enabling data preprocessing 150 | do_preprocess=True, 151 | ) 152 | .update( 153 | iris, 154 | data, 155 | preprocess_data, 156 | investigate_data, 157 | split_data, 158 | training_features, 159 | training_target, 160 | test_features, 161 | test_target, 162 | model, 163 | predictions, 164 | classification_metrics, 165 | confusion_matrix, 166 | ) 167 | ) 168 | -------------------------------------------------------------------------------- /fn_graph/examples/namespaces.py: -------------------------------------------------------------------------------- 1 | "A basic example of namespaces." 2 | #%% 3 | from fn_graph import Composer 4 | 5 | # Namespaces example 6 | 7 | 8 | def data(): 9 | return 5 10 | 11 | 12 | def b(data, factor): 13 | return data * factor 14 | 15 | 16 | def c(b): 17 | return b 18 | 19 | 20 | def combined_result(child_one__c, child_two__c): 21 | pass 22 | 23 | 24 | child = Composer().update(b, c) 25 | parent = ( 26 | Composer() 27 | .update_namespaces(child_one=child, child_two=child) 28 | .update(data, combined_result) 29 | .update_parameters(child_one__factor=3, child_two__factor=5) 30 | ) 31 | 32 | # %% 33 | parent.graphviz() 34 | 35 | # %% 36 | 37 | # Link example 38 | 39 | 40 | def calculated_factor(data): 41 | return data / 2 42 | 43 | 44 | factor_calc = Composer() 45 | factoring = Composer().update(calculated_factor) 46 | 47 | linked_parent = ( 48 | Composer() 49 | .update_namespaces(child_one=child, factoring=factoring) 50 | .update(data) 51 | .link(child_one__factor="factoring__calculated_factor") 52 | ) 53 | 54 | f = linked_parent 55 | 56 | # %% 57 | -------------------------------------------------------------------------------- /fn_graph/examples/plotting.py: -------------------------------------------------------------------------------- 1 | "Examples of various different supported plotting libraries." 2 | #%% 3 | import math 4 | 5 | import matplotlib.pylab as plt 6 | import plotly.express as px 7 | import seaborn as sns 8 | from fn_graph import Composer 9 | 10 | 11 | def matplotlib_plot(data): 12 | fig2, ax = plt.subplots() 13 | ax.plot(data.total_bill, data.tip, "o") 14 | return fig2 15 | 16 | 17 | def seaborn_plot(data): 18 | 19 | return sns.relplot( 20 | x="total_bill", 21 | y="tip", 22 | col="time", 23 | hue="smoker", 24 | style="smoker", 25 | size="size", 26 | data=data, 27 | ) 28 | 29 | 30 | def pandas_plot(data): 31 | return data.total_bill.plot.hist() 32 | 33 | 34 | def plotly_plot(data): 35 | return px.scatter( 36 | data, 37 | x="total_bill", 38 | y="tip", 39 | color="sex", 40 | facet_row="time", 41 | facet_col="day", 42 | trendline="ols", 43 | size="size", 44 | symbol="smoker", 45 | ) 46 | 47 | 48 | f = ( 49 | Composer() 50 | .update_parameters(data=sns.load_dataset("tips")) 51 | .update(matplotlib_plot, seaborn_plot, pandas_plot, plotly_plot) 52 | ) 53 | -------------------------------------------------------------------------------- /fn_graph/examples/share_prices.csv: -------------------------------------------------------------------------------- 1 | datetime,share_code,share_price 2 | 2010-01-04T00:00:00,CZA,1300.0 3 | 2010-01-04T00:00:00,GBG,1294.0 4 | 2010-01-04T00:00:00,MRF,141.0 5 | 2010-01-04T00:00:00,GDO,232.0 6 | 2010-01-05T00:00:00,CZA,1348.0 7 | 2010-01-05T00:00:00,GBG,1325.0 8 | 2010-01-05T00:00:00,MRF,138.0 9 | 2010-01-06T00:00:00,CZA,1385.0 10 | 2010-01-06T00:00:00,GBG,1350.0 11 | 2010-01-06T00:00:00,GDO,231.0 12 | 2010-01-06T00:00:00,MRF,136.0 13 | 2010-01-07T00:00:00,CZA,1430.0 14 | 2010-01-07T00:00:00,GBG,1375.0 15 | 2010-01-07T00:00:00,GDO,234.0 16 | 2010-01-08T00:00:00,CZA,1500.0 17 | 2010-01-08T00:00:00,GBG,1415.0 18 | 2010-01-08T00:00:00,GDO,232.0 19 | 2010-01-08T00:00:00,MRF,142.0 20 | 2010-01-11T00:00:00,CZA,1506.0 21 | 2010-01-11T00:00:00,GBG,1475.0 22 | 2010-01-11T00:00:00,GDO,238.0 23 | 2010-01-12T00:00:00,CZA,1441.0 24 | 2010-01-12T00:00:00,GBG,1440.0 25 | 2010-01-12T00:00:00,GDO,242.0 26 | 2010-01-12T00:00:00,MRF,143.0 27 | 2010-01-13T00:00:00,CZA,1411.0 28 | 2010-01-13T00:00:00,GBG,1390.0 29 | 2010-01-13T00:00:00,GDO,229.0 30 | 2010-01-13T00:00:00,MRF,142.0 31 | 2010-01-14T00:00:00,CZA,1400.0 32 | 2010-01-14T00:00:00,GBG,1404.0 33 | 2010-01-14T00:00:00,GDO,235.0 34 | 2010-01-15T00:00:00,CZA,1460.0 35 | 2010-01-15T00:00:00,GBG,1400.0 36 | 2010-01-15T00:00:00,GDO,237.0 37 | 2010-01-18T00:00:00,CZA,1450.0 38 | 2010-01-18T00:00:00,GBG,1365.0 39 | 2010-01-18T00:00:00,GDO,240.0 40 | 2010-01-18T00:00:00,MRF,140.0 41 | 2010-01-19T00:00:00,CZA,1433.0 42 | 2010-01-19T00:00:00,GDO,230.0 43 | 2010-01-19T00:00:00,MRF,138.0 44 | 2010-01-19T00:00:00,GBG,1400.0 45 | 2010-01-20T00:00:00,CZA,1501.0 46 | 2010-01-20T00:00:00,GDO,231.0 47 | 2010-01-20T00:00:00,MRF,135.0 48 | 2010-01-21T00:00:00,CZA,1450.0 49 | 2010-01-21T00:00:00,GDO,219.0 50 | 2010-01-21T00:00:00,MRF,137.0 51 | 2010-01-21T00:00:00,GBG,1390.0 52 | 2010-01-22T00:00:00,CZA,1480.0 53 | 2010-01-22T00:00:00,GDO,228.0 54 | 2010-01-22T00:00:00,MRF,131.0 55 | 2010-01-25T00:00:00,CZA,1515.0 56 | 2010-01-25T00:00:00,MRF,132.0 57 | 2010-01-25T00:00:00,GBG,1375.0 58 | 2010-01-25T00:00:00,GDO,218.0 59 | 2010-01-26T00:00:00,CZA,1490.0 60 | 2010-01-26T00:00:00,MRF,131.0 61 | 2010-01-27T00:00:00,CZA,1480.0 62 | 2010-01-27T00:00:00,MRF,130.0 63 | 2010-01-27T00:00:00,GBG,1322.0 64 | 2010-01-27T00:00:00,GDO,220.0 65 | 2010-01-28T00:00:00,CZA,1490.0 66 | 2010-01-28T00:00:00,MRF,133.0 67 | 2010-01-29T00:00:00,CZA,1455.0 68 | 2010-01-29T00:00:00,GBG,1270.0 69 | 2010-01-29T00:00:00,GDO,218.0 70 | 2010-01-29T00:00:00,MRF,130.0 71 | 2010-02-01T00:00:00,CZA,1420.0 72 | 2010-02-01T00:00:00,GBG,1260.0 73 | 2010-02-01T00:00:00,GDO,215.0 74 | 2010-02-01T00:00:00,MRF,131.0 75 | 2010-02-02T00:00:00,CZA,1585.0 76 | 2010-02-02T00:00:00,GBG,1298.0 77 | 2010-02-02T00:00:00,MRF,134.0 78 | 2010-02-02T00:00:00,GDO,208.0 79 | 2010-02-03T00:00:00,CZA,1625.0 80 | 2010-02-03T00:00:00,GBG,1330.0 81 | 2010-02-03T00:00:00,MRF,135.0 82 | 2010-02-04T00:00:00,CZA,1580.0 83 | 2010-02-04T00:00:00,GBG,1310.0 84 | 2010-02-04T00:00:00,GDO,210.0 85 | 2010-02-05T00:00:00,CZA,1605.0 86 | 2010-02-05T00:00:00,GBG,1242.0 87 | 2010-02-05T00:00:00,MRF,130.0 88 | 2010-02-08T00:00:00,CZA,1600.0 89 | 2010-02-08T00:00:00,GBG,1300.0 90 | 2010-02-08T00:00:00,GDO,209.0 91 | 2010-02-08T00:00:00,MRF,128.0 92 | 2010-02-09T00:00:00,CZA,1640.0 93 | 2010-02-09T00:00:00,GBG,1330.0 94 | 2010-02-09T00:00:00,GDO,205.0 95 | 2010-02-09T00:00:00,MRF,130.0 96 | 2010-02-10T00:00:00,CZA,1601.0 97 | 2010-02-10T00:00:00,GBG,1270.0 98 | 2010-02-10T00:00:00,GDO,209.0 99 | 2010-02-10T00:00:00,MRF,132.0 100 | 2010-02-11T00:00:00,CZA,1600.0 101 | 2010-02-12T00:00:00,CZA,1569.0 102 | 2010-02-12T00:00:00,GBG,1300.0 103 | 2010-02-12T00:00:00,MRF,133.0 104 | 2010-02-12T00:00:00,GDO,210.0 105 | 2010-02-15T00:00:00,CZA,1549.0 106 | 2010-02-15T00:00:00,GBG,1235.0 107 | 2010-02-15T00:00:00,MRF,136.0 108 | 2010-02-16T00:00:00,CZA,1640.0 109 | 2010-02-16T00:00:00,MRF,137.0 110 | 2010-02-16T00:00:00,GBG,1340.0 111 | 2010-02-17T00:00:00,CZA,1601.0 112 | 2010-02-17T00:00:00,GDO,209.0 113 | 2010-02-17T00:00:00,MRF,142.0 114 | 2010-02-18T00:00:00,CZA,1750.0 115 | 2010-02-18T00:00:00,GDO,205.0 116 | 2010-02-18T00:00:00,GBG,1302.0 117 | 2010-02-19T00:00:00,CZA,1740.0 118 | 2010-02-19T00:00:00,GDO,204.0 119 | 2010-02-22T00:00:00,CZA,1759.0 120 | 2010-02-22T00:00:00,GBG,1252.0 121 | 2010-02-22T00:00:00,GDO,200.0 122 | 2010-02-22T00:00:00,MRF,144.0 123 | 2010-02-23T00:00:00,CZA,1694.0 124 | 2010-02-23T00:00:00,GBG,1235.0 125 | 2010-02-23T00:00:00,GDO,199.0 126 | 2010-02-23T00:00:00,MRF,143.0 127 | 2010-02-24T00:00:00,CZA,1734.0 128 | 2010-02-24T00:00:00,GDO,193.0 129 | 2010-02-24T00:00:00,MRF,142.0 130 | 2010-02-24T00:00:00,GBG,1225.0 131 | 2010-02-25T00:00:00,CZA,1690.0 132 | 2010-02-25T00:00:00,GDO,190.0 133 | 2010-02-25T00:00:00,MRF,140.0 134 | 2010-02-26T00:00:00,CZA,1654.0 135 | 2010-02-26T00:00:00,GDO,199.0 136 | 2010-02-26T00:00:00,MRF,145.0 137 | 2010-03-01T00:00:00,CZA,1621.0 138 | 2010-03-01T00:00:00,GBG,1270.0 139 | 2010-03-02T00:00:00,CZA,1620.0 140 | 2010-03-02T00:00:00,GBG,1293.0 141 | 2010-03-02T00:00:00,MRF,143.0 142 | 2010-03-02T00:00:00,GDO,195.0 143 | 2010-03-03T00:00:00,CZA,1624.0 144 | 2010-03-03T00:00:00,MRF,150.0 145 | 2010-03-03T00:00:00,GBG,1295.0 146 | 2010-03-04T00:00:00,CZA,1616.0 147 | 2010-03-04T00:00:00,GDO,194.0 148 | 2010-03-04T00:00:00,MRF,151.0 149 | 2010-03-05T00:00:00,CZA,1679.0 150 | 2010-03-05T00:00:00,GDO,195.0 151 | 2010-03-05T00:00:00,MRF,159.0 152 | 2010-03-08T00:00:00,CZA,1681.0 153 | 2010-03-08T00:00:00,GDO,194.0 154 | 2010-03-08T00:00:00,MRF,161.0 155 | 2010-03-08T00:00:00,GBG,1278.0 156 | 2010-03-09T00:00:00,CZA,1646.0 157 | 2010-03-09T00:00:00,GDO,179.0 158 | 2010-03-09T00:00:00,MRF,154.0 159 | 2010-03-10T00:00:00,CZA,1629.0 160 | 2010-03-10T00:00:00,GBG,1265.0 161 | 2010-03-10T00:00:00,GDO,192.0 162 | 2010-03-10T00:00:00,MRF,155.0 163 | 2010-03-11T00:00:00,CZA,1630.0 164 | 2010-03-11T00:00:00,GBG,1269.0 165 | 2010-03-11T00:00:00,GDO,185.0 166 | 2010-03-11T00:00:00,MRF,154.0 167 | 2010-03-12T00:00:00,CZA,1610.0 168 | 2010-03-12T00:00:00,GBG,1270.0 169 | 2010-03-12T00:00:00,MRF,156.0 170 | 2010-03-12T00:00:00,GDO,195.0 171 | 2010-03-15T00:00:00,CZA,1550.0 172 | 2010-03-15T00:00:00,GBG,1280.0 173 | 2010-03-15T00:00:00,MRF,153.0 174 | 2010-03-16T00:00:00,CZA,1470.0 175 | 2010-03-16T00:00:00,GBG,1240.0 176 | 2010-03-17T00:00:00,CZA,1458.0 177 | 2010-03-17T00:00:00,GDO,192.0 178 | 2010-03-17T00:00:00,MRF,152.0 179 | 2010-03-17T00:00:00,GBG,1250.0 180 | 2010-03-18T00:00:00,CZA,1495.0 181 | 2010-03-18T00:00:00,GDO,199.0 182 | 2010-03-18T00:00:00,MRF,150.0 183 | 2010-03-19T00:00:00,CZA,1485.0 184 | 2010-03-19T00:00:00,GDO,200.0 185 | 2010-03-23T00:00:00,CZA,1525.0 186 | 2010-03-23T00:00:00,GDO,195.0 187 | 2010-03-23T00:00:00,MRF,159.0 188 | 2010-03-24T00:00:00,CZA,1565.0 189 | 2010-03-24T00:00:00,GDO,192.0 190 | 2010-03-24T00:00:00,MRF,160.0 191 | 2010-03-25T00:00:00,CZA,1620.0 192 | 2010-03-25T00:00:00,GBG,1255.0 193 | 2010-03-25T00:00:00,GDO,190.0 194 | 2010-03-25T00:00:00,MRF,166.0 195 | 2010-03-26T00:00:00,CZA,1555.0 196 | 2010-03-26T00:00:00,GBG,1228.0 197 | 2010-03-26T00:00:00,GDO,195.0 198 | 2010-03-26T00:00:00,MRF,163.0 199 | 2010-03-29T00:00:00,CZA,1600.0 200 | 2010-03-29T00:00:00,GDO,198.0 201 | 2010-03-29T00:00:00,GBG,1250.0 202 | 2010-03-30T00:00:00,CZA,1590.0 203 | 2010-03-30T00:00:00,GDO,195.0 204 | 2010-03-30T00:00:00,MRF,164.0 205 | 2010-03-31T00:00:00,CZA,1591.0 206 | 2010-04-01T00:00:00,CZA,1603.0 207 | 2010-04-01T00:00:00,GDO,193.0 208 | 2010-04-01T00:00:00,MRF,178.0 209 | 2010-04-06T00:00:00,CZA,1600.0 210 | 2010-04-06T00:00:00,GBG,1270.0 211 | 2010-04-06T00:00:00,GDO,188.0 212 | 2010-04-06T00:00:00,MRF,187.0 213 | 2010-04-07T00:00:00,CZA,1601.0 214 | 2010-04-07T00:00:00,GBG,1264.0 215 | 2010-04-07T00:00:00,GDO,190.0 216 | 2010-04-07T00:00:00,MRF,186.0 217 | 2010-04-08T00:00:00,CZA,1570.0 218 | 2010-04-08T00:00:00,GBG,1299.0 219 | 2010-04-08T00:00:00,GDO,198.0 220 | 2010-04-08T00:00:00,MRF,179.0 221 | 2010-04-09T00:00:00,CZA,1620.0 222 | 2010-04-09T00:00:00,GDO,195.0 223 | 2010-04-09T00:00:00,GBG,1270.0 224 | 2010-04-09T00:00:00,MRF,186.0 225 | 2010-04-12T00:00:00,CZA,1653.0 226 | 2010-04-12T00:00:00,GDO,190.0 227 | 2010-04-13T00:00:00,CZA,1710.0 228 | 2010-04-13T00:00:00,GBG,1250.0 229 | 2010-04-13T00:00:00,MRF,180.0 230 | 2010-04-14T00:00:00,CZA,1753.0 231 | 2010-04-14T00:00:00,GDO,189.0 232 | 2010-04-14T00:00:00,MRF,187.0 233 | 2010-04-14T00:00:00,GBG,1270.0 234 | 2010-04-15T00:00:00,CZA,1730.0 235 | 2010-04-15T00:00:00,GDO,188.0 236 | 2010-04-15T00:00:00,MRF,190.0 237 | 2010-04-16T00:00:00,CZA,1760.0 238 | 2010-04-16T00:00:00,GDO,190.0 239 | 2010-04-16T00:00:00,GBG,1250.0 240 | 2010-04-19T00:00:00,CZA,1715.0 241 | 2010-04-19T00:00:00,GDO,195.0 242 | 2010-04-19T00:00:00,MRF,185.0 243 | 2010-04-20T00:00:00,CZA,1750.0 244 | 2010-04-20T00:00:00,GDO,190.0 245 | 2010-04-20T00:00:00,GBG,1300.0 246 | 2010-04-21T00:00:00,CZA,1735.0 247 | 2010-04-21T00:00:00,GDO,193.0 248 | 2010-04-21T00:00:00,MRF,181.0 249 | 2010-04-22T00:00:00,CZA,1740.0 250 | 2010-04-22T00:00:00,MRF,178.0 251 | 2010-04-22T00:00:00,GDO,194.0 252 | 2010-04-23T00:00:00,MRF,177.0 253 | 2010-04-23T00:00:00,CZA,1750.0 254 | 2010-04-26T00:00:00,GBG,1295.0 255 | 2010-04-26T00:00:00,MRF,184.0 256 | 2010-04-28T00:00:00,CZA,1696.0 257 | 2010-04-28T00:00:00,GBG,1300.0 258 | 2010-04-28T00:00:00,GDO,193.0 259 | 2010-04-28T00:00:00,MRF,178.0 260 | 2010-04-29T00:00:00,CZA,1667.0 261 | 2010-04-29T00:00:00,GBG,1351.0 262 | 2010-04-29T00:00:00,GDO,190.0 263 | 2010-04-30T00:00:00,CZA,1635.0 264 | 2010-04-30T00:00:00,GDO,194.0 265 | 2010-04-30T00:00:00,MRF,183.0 266 | 2010-04-30T00:00:00,GBG,1400.0 267 | 2010-05-03T00:00:00,CZA,1594.0 268 | 2010-05-03T00:00:00,GDO,190.0 269 | 2010-05-03T00:00:00,MRF,180.0 270 | 2010-05-04T00:00:00,CZA,1475.0 271 | 2010-05-04T00:00:00,GBG,1390.0 272 | 2010-05-04T00:00:00,GDO,186.0 273 | 2010-05-04T00:00:00,MRF,169.0 274 | 2010-05-05T00:00:00,CZA,1409.0 275 | 2010-05-05T00:00:00,GBG,1365.0 276 | 2010-05-05T00:00:00,GDO,180.0 277 | 2010-05-06T00:00:00,CZA,1450.0 278 | 2010-05-06T00:00:00,GBG,1400.0 279 | 2010-05-06T00:00:00,GDO,177.0 280 | 2010-05-06T00:00:00,MRF,171.0 281 | 2010-05-07T00:00:00,CZA,1395.0 282 | 2010-05-07T00:00:00,GBG,1341.0 283 | 2010-05-07T00:00:00,GDO,185.0 284 | 2010-05-07T00:00:00,MRF,158.0 285 | 2010-05-10T00:00:00,CZA,1480.0 286 | 2010-05-10T00:00:00,GBG,1270.0 287 | 2010-05-10T00:00:00,GDO,180.0 288 | 2010-05-10T00:00:00,MRF,171.0 289 | 2010-05-11T00:00:00,CZA,1476.0 290 | 2010-05-11T00:00:00,GBG,1370.0 291 | 2010-05-11T00:00:00,GDO,185.0 292 | 2010-05-11T00:00:00,MRF,166.0 293 | 2010-05-12T00:00:00,CZA,1446.0 294 | 2010-05-12T00:00:00,GBG,1425.0 295 | 2010-05-12T00:00:00,GDO,184.0 296 | 2010-05-12T00:00:00,MRF,170.0 297 | 2010-05-13T00:00:00,CZA,1451.0 298 | 2010-05-13T00:00:00,GDO,186.0 299 | 2010-05-13T00:00:00,MRF,169.0 300 | 2010-05-13T00:00:00,GBG,1424.0 301 | 2010-05-14T00:00:00,CZA,1392.0 302 | 2010-05-14T00:00:00,MRF,168.0 303 | 2010-05-14T00:00:00,GDO,199.0 304 | 2010-05-17T00:00:00,CZA,1432.0 305 | 2010-05-17T00:00:00,MRF,164.0 306 | 2010-05-18T00:00:00,CZA,1475.0 307 | 2010-05-18T00:00:00,GDO,195.0 308 | 2010-05-18T00:00:00,MRF,168.0 309 | 2010-05-19T00:00:00,CZA,1400.0 310 | 2010-05-19T00:00:00,GBG,1320.0 311 | 2010-05-19T00:00:00,GDO,192.0 312 | 2010-05-19T00:00:00,MRF,166.0 313 | 2010-05-20T00:00:00,CZA,1300.0 314 | 2010-05-20T00:00:00,GDO,189.0 315 | 2010-05-20T00:00:00,MRF,160.0 316 | 2010-05-20T00:00:00,GBG,1300.0 317 | 2010-05-21T00:00:00,CZA,1275.0 318 | 2010-05-21T00:00:00,GDO,188.0 319 | 2010-05-21T00:00:00,MRF,159.0 320 | 2010-05-24T00:00:00,CZA,1379.0 321 | 2010-05-24T00:00:00,GDO,184.0 322 | 2010-05-24T00:00:00,MRF,162.0 323 | 2010-05-25T00:00:00,CZA,1200.0 324 | 2010-05-25T00:00:00,GBG,1271.0 325 | 2010-05-25T00:00:00,GDO,183.0 326 | 2010-05-25T00:00:00,MRF,153.0 327 | 2010-05-26T00:00:00,CZA,1215.0 328 | 2010-05-26T00:00:00,GBG,1310.0 329 | 2010-05-26T00:00:00,GDO,179.0 330 | 2010-05-26T00:00:00,MRF,160.0 331 | 2010-05-27T00:00:00,CZA,1299.0 332 | 2010-05-27T00:00:00,GBG,1240.0 333 | 2010-05-27T00:00:00,MRF,165.0 334 | 2010-05-27T00:00:00,GDO,187.0 335 | 2010-05-28T00:00:00,CZA,1257.0 336 | 2010-05-28T00:00:00,MRF,162.0 337 | 2010-05-28T00:00:00,GBG,1295.0 338 | 2010-05-31T00:00:00,CZA,1315.0 339 | 2010-05-31T00:00:00,GDO,181.0 340 | 2010-05-31T00:00:00,MRF,163.0 341 | 2010-06-01T00:00:00,CZA,1277.0 342 | 2010-06-01T00:00:00,GBG,1273.0 343 | 2010-06-01T00:00:00,MRF,162.0 344 | 2010-06-01T00:00:00,GDO,184.0 345 | 2010-06-02T00:00:00,CZA,1255.0 346 | 2010-06-02T00:00:00,MRF,160.0 347 | 2010-06-02T00:00:00,GBG,1282.0 348 | 2010-06-03T00:00:00,CZA,1305.0 349 | 2010-06-03T00:00:00,GDO,183.0 350 | 2010-06-03T00:00:00,MRF,156.0 351 | 2010-06-04T00:00:00,CZA,1293.0 352 | 2010-06-04T00:00:00,GBG,1356.0 353 | 2010-06-04T00:00:00,GDO,180.0 354 | 2010-06-04T00:00:00,MRF,147.0 355 | 2010-06-07T00:00:00,CZA,1253.0 356 | 2010-06-07T00:00:00,GBG,1276.0 357 | 2010-06-07T00:00:00,GDO,181.0 358 | 2010-06-07T00:00:00,MRF,136.0 359 | 2010-06-08T00:00:00,CZA,1250.0 360 | 2010-06-08T00:00:00,GBG,1300.0 361 | 2010-06-08T00:00:00,GDO,180.0 362 | 2010-06-08T00:00:00,MRF,134.0 363 | 2010-06-09T00:00:00,CZA,1225.0 364 | 2010-06-09T00:00:00,GBG,1310.0 365 | 2010-06-09T00:00:00,GDO,184.0 366 | 2010-06-10T00:00:00,CZA,1190.0 367 | 2010-06-10T00:00:00,MRF,141.0 368 | 2010-06-10T00:00:00,GBG,1320.0 369 | 2010-06-10T00:00:00,GDO,182.0 370 | 2010-06-11T00:00:00,CZA,1210.0 371 | 2010-06-11T00:00:00,MRF,139.0 372 | 2010-06-14T00:00:00,CZA,1250.0 373 | 2010-06-14T00:00:00,GBG,1316.0 374 | 2010-06-14T00:00:00,MRF,142.0 375 | 2010-06-15T00:00:00,CZA,1265.0 376 | 2010-06-15T00:00:00,GBG,1333.0 377 | 2010-06-15T00:00:00,GDO,180.0 378 | 2010-06-17T00:00:00,CZA,1266.0 379 | 2010-06-17T00:00:00,GBG,1343.0 380 | 2010-06-17T00:00:00,GDO,184.0 381 | 2010-06-17T00:00:00,MRF,140.0 382 | 2010-06-18T00:00:00,CZA,1265.0 383 | 2010-06-18T00:00:00,GBG,1375.0 384 | 2010-06-18T00:00:00,GDO,187.0 385 | 2010-06-18T00:00:00,MRF,139.0 386 | 2010-06-21T00:00:00,CZA,1283.0 387 | 2010-06-21T00:00:00,GDO,192.0 388 | 2010-06-21T00:00:00,MRF,143.0 389 | 2010-06-21T00:00:00,GBG,1380.0 390 | 2010-06-22T00:00:00,CZA,1276.0 391 | 2010-06-22T00:00:00,GDO,193.0 392 | 2010-06-22T00:00:00,MRF,142.0 393 | 2010-06-23T00:00:00,CZA,1256.0 394 | 2010-06-23T00:00:00,GBG,1342.0 395 | 2010-06-23T00:00:00,GDO,200.0 396 | 2010-06-23T00:00:00,MRF,139.0 397 | 2010-06-24T00:00:00,CZA,1250.0 398 | 2010-06-24T00:00:00,GBG,1370.0 399 | 2010-06-24T00:00:00,GDO,199.0 400 | 2010-06-25T00:00:00,CZA,1244.0 401 | 2010-06-25T00:00:00,GBG,1380.0 402 | 2010-06-25T00:00:00,GDO,194.0 403 | 2010-06-28T00:00:00,CZA,1201.0 404 | 2010-06-28T00:00:00,GBG,1390.0 405 | 2010-06-28T00:00:00,GDO,198.0 406 | 2010-06-28T00:00:00,MRF,136.0 407 | 2010-06-29T00:00:00,CZA,1180.0 408 | 2010-06-29T00:00:00,GDO,187.0 409 | 2010-06-29T00:00:00,MRF,129.0 410 | 2010-06-29T00:00:00,GBG,1360.0 411 | 2010-06-30T00:00:00,CZA,1150.0 412 | 2010-06-30T00:00:00,GDO,182.0 413 | 2010-06-30T00:00:00,MRF,125.0 414 | 2010-07-01T00:00:00,CZA,1095.0 415 | 2010-07-01T00:00:00,GBG,1305.0 416 | 2010-07-01T00:00:00,GDO,185.0 417 | 2010-07-01T00:00:00,MRF,117.0 418 | 2010-07-02T00:00:00,CZA,1140.0 419 | 2010-07-02T00:00:00,GDO,184.0 420 | 2010-07-02T00:00:00,MRF,120.0 421 | 2010-07-02T00:00:00,GBG,1310.0 422 | 2010-07-05T00:00:00,CZA,1076.0 423 | 2010-07-05T00:00:00,MRF,121.0 424 | 2010-07-05T00:00:00,GDO,180.0 425 | 2010-07-06T00:00:00,CZA,1105.0 426 | 2010-07-06T00:00:00,MRF,129.0 427 | 2010-07-06T00:00:00,GBG,1250.0 428 | 2010-07-07T00:00:00,CZA,1149.0 429 | 2010-07-07T00:00:00,MRF,131.0 430 | 2010-07-08T00:00:00,CZA,1220.0 431 | 2010-07-08T00:00:00,GBG,1300.0 432 | 2010-07-08T00:00:00,GDO,178.0 433 | 2010-07-09T00:00:00,CZA,1270.0 434 | 2010-07-09T00:00:00,GBG,1370.0 435 | 2010-07-09T00:00:00,GDO,175.0 436 | 2010-07-09T00:00:00,MRF,132.0 437 | 2010-07-12T00:00:00,CZA,1280.0 438 | 2010-07-12T00:00:00,MRF,131.0 439 | 2010-07-12T00:00:00,GDO,178.0 440 | 2010-07-12T00:00:00,GBG,1375.0 441 | 2010-07-13T00:00:00,CZA,1290.0 442 | 2010-07-13T00:00:00,MRF,130.0 443 | 2010-07-14T00:00:00,CZA,1265.0 444 | 2010-07-14T00:00:00,GDO,179.0 445 | 2010-07-15T00:00:00,CZA,1275.0 446 | 2010-07-15T00:00:00,MRF,135.0 447 | 2010-07-15T00:00:00,GBG,1350.0 448 | 2010-07-16T00:00:00,CZA,1294.0 449 | 2010-07-16T00:00:00,GDO,178.0 450 | 2010-07-16T00:00:00,MRF,134.0 451 | 2010-07-19T00:00:00,CZA,1290.0 452 | 2010-07-19T00:00:00,GDO,172.0 453 | 2010-07-19T00:00:00,MRF,130.0 454 | 2010-07-20T00:00:00,CZA,1258.0 455 | 2010-07-20T00:00:00,GBG,1300.0 456 | 2010-07-20T00:00:00,MRF,129.0 457 | 2010-07-20T00:00:00,GDO,168.0 458 | 2010-07-21T00:00:00,CZA,1285.0 459 | 2010-07-21T00:00:00,GBG,1310.0 460 | 2010-07-21T00:00:00,MRF,137.0 461 | 2010-07-22T00:00:00,CZA,1277.0 462 | 2010-07-22T00:00:00,GBG,1305.0 463 | 2010-07-22T00:00:00,GDO,171.0 464 | 2010-07-22T00:00:00,MRF,134.0 465 | 2010-07-23T00:00:00,CZA,1289.0 466 | 2010-07-23T00:00:00,GBG,1324.0 467 | 2010-07-23T00:00:00,GDO,169.0 468 | 2010-07-26T00:00:00,CZA,1255.0 469 | 2010-07-26T00:00:00,GBG,1280.0 470 | 2010-07-26T00:00:00,MRF,135.0 471 | 2010-07-26T00:00:00,GDO,165.0 472 | 2010-07-27T00:00:00,CZA,1289.0 473 | 2010-07-27T00:00:00,GBG,1270.0 474 | 2010-07-27T00:00:00,MRF,134.0 475 | 2010-07-28T00:00:00,CZA,1259.0 476 | 2010-07-28T00:00:00,GBG,1314.0 477 | 2010-07-28T00:00:00,GDO,174.0 478 | 2010-07-28T00:00:00,MRF,136.0 479 | 2010-07-29T00:00:00,CZA,1210.0 480 | 2010-07-29T00:00:00,GDO,179.0 481 | 2010-07-29T00:00:00,MRF,137.0 482 | 2010-07-29T00:00:00,GBG,1305.0 483 | 2010-07-30T00:00:00,CZA,1090.0 484 | 2010-07-30T00:00:00,GDO,178.0 485 | 2010-08-02T00:00:00,CZA,1170.0 486 | 2010-08-02T00:00:00,MRF,143.0 487 | 2010-08-03T00:00:00,CZA,1171.0 488 | 2010-08-03T00:00:00,GBG,1275.0 489 | 2010-08-03T00:00:00,GDO,175.0 490 | 2010-08-04T00:00:00,CZA,1165.0 491 | 2010-08-04T00:00:00,MRF,140.0 492 | 2010-08-04T00:00:00,GBG,1276.0 493 | 2010-08-05T00:00:00,CZA,1086.0 494 | 2010-08-05T00:00:00,GDO,170.0 495 | 2010-08-05T00:00:00,MRF,142.0 496 | 2010-08-06T00:00:00,CZA,1096.0 497 | 2010-08-06T00:00:00,MRF,141.0 498 | 2010-08-06T00:00:00,GDO,175.0 499 | 2010-08-10T00:00:00,CZA,1115.0 500 | 2010-08-10T00:00:00,GBG,1290.0 501 | 2010-08-10T00:00:00,MRF,135.0 502 | 2010-08-11T00:00:00,CZA,955.0 503 | 2010-08-11T00:00:00,GBG,1320.0 504 | 2010-08-11T00:00:00,MRF,130.0 505 | 2010-08-12T00:00:00,CZA,895.0 506 | 2010-08-12T00:00:00,GBG,1340.0 507 | 2010-08-12T00:00:00,MRF,134.0 508 | 2010-08-13T00:00:00,CZA,890.0 509 | 2010-08-13T00:00:00,GBG,1346.0 510 | 2010-08-13T00:00:00,GDO,174.0 511 | 2010-08-13T00:00:00,MRF,136.0 512 | 2010-08-16T00:00:00,CZA,925.0 513 | 2010-08-16T00:00:00,GBG,1379.0 514 | 2010-08-16T00:00:00,GDO,175.0 515 | 2010-08-16T00:00:00,MRF,138.0 516 | 2010-08-17T00:00:00,CZA,970.0 517 | 2010-08-17T00:00:00,GBG,1440.0 518 | 2010-08-18T00:00:00,CZA,956.0 519 | 2010-08-18T00:00:00,GDO,180.0 520 | 2010-08-18T00:00:00,MRF,135.0 521 | 2010-08-19T00:00:00,CZA,945.0 522 | 2010-08-19T00:00:00,GBG,1550.0 523 | 2010-08-19T00:00:00,GDO,184.0 524 | 2010-08-19T00:00:00,MRF,132.0 525 | 2010-08-20T00:00:00,CZA,900.0 526 | 2010-08-20T00:00:00,GDO,195.0 527 | 2010-08-20T00:00:00,MRF,130.0 528 | 2010-08-20T00:00:00,GBG,1500.0 529 | 2010-08-23T00:00:00,CZA,911.0 530 | 2010-08-23T00:00:00,GDO,194.0 531 | 2010-08-23T00:00:00,MRF,132.0 532 | 2010-08-24T00:00:00,CZA,875.0 533 | 2010-08-24T00:00:00,GBG,1530.0 534 | 2010-08-24T00:00:00,GDO,190.0 535 | 2010-08-24T00:00:00,MRF,130.0 536 | 2010-08-25T00:00:00,CZA,880.0 537 | 2010-08-25T00:00:00,GBG,1540.0 538 | 2010-08-25T00:00:00,GDO,194.0 539 | 2010-08-25T00:00:00,MRF,126.0 540 | 2010-08-26T00:00:00,CZA,885.0 541 | 2010-08-26T00:00:00,GBG,1565.0 542 | 2010-08-26T00:00:00,GDO,192.0 543 | 2010-08-26T00:00:00,MRF,130.0 544 | 2010-08-27T00:00:00,CZA,874.0 545 | 2010-08-27T00:00:00,GBG,1536.0 546 | 2010-08-27T00:00:00,GDO,197.0 547 | 2010-08-27T00:00:00,MRF,128.0 548 | 2010-08-30T00:00:00,CZA,950.0 549 | 2010-08-30T00:00:00,GBG,1541.0 550 | 2010-08-30T00:00:00,GDO,195.0 551 | 2010-08-30T00:00:00,MRF,129.0 552 | 2010-08-31T00:00:00,GBG,1570.0 553 | 2010-08-31T00:00:00,GDO,189.0 554 | 2010-08-31T00:00:00,MRF,124.0 555 | 2010-08-31T00:00:00,CZA,900.0 556 | 2010-09-01T00:00:00,GBG,1550.0 557 | 2010-09-01T00:00:00,MRF,129.0 558 | 2010-09-01T00:00:00,GDO,188.0 559 | 2010-09-02T00:00:00,CZA,890.0 560 | 2010-09-02T00:00:00,GBG,1501.0 561 | 2010-09-02T00:00:00,MRF,128.0 562 | 2010-09-03T00:00:00,CZA,968.0 563 | 2010-09-03T00:00:00,GBG,1510.0 564 | 2010-09-03T00:00:00,GDO,187.0 565 | 2010-09-03T00:00:00,MRF,131.0 566 | 2010-09-06T00:00:00,CZA,930.0 567 | 2010-09-06T00:00:00,GBG,1550.0 568 | 2010-09-06T00:00:00,GDO,188.0 569 | 2010-09-06T00:00:00,MRF,132.0 570 | 2010-09-07T00:00:00,CZA,947.0 571 | 2010-09-07T00:00:00,GBG,1691.0 572 | 2010-09-07T00:00:00,GDO,187.0 573 | 2010-09-07T00:00:00,MRF,128.0 574 | 2010-09-08T00:00:00,CZA,940.0 575 | 2010-09-08T00:00:00,GBG,1744.0 576 | 2010-09-09T00:00:00,CZA,902.0 577 | 2010-09-09T00:00:00,GBG,1750.0 578 | 2010-09-09T00:00:00,GDO,194.0 579 | 2010-09-09T00:00:00,MRF,127.0 580 | 2010-09-10T00:00:00,CZA,901.0 581 | 2010-09-10T00:00:00,GBG,1761.0 582 | 2010-09-10T00:00:00,GDO,195.0 583 | 2010-09-10T00:00:00,MRF,129.0 584 | 2010-09-13T00:00:00,CZA,992.0 585 | 2010-09-13T00:00:00,GBG,1700.0 586 | 2010-09-13T00:00:00,GDO,194.0 587 | 2010-09-13T00:00:00,MRF,137.0 588 | 2010-09-14T00:00:00,CZA,1006.0 589 | 2010-09-14T00:00:00,GBG,1734.0 590 | 2010-09-14T00:00:00,GDO,198.0 591 | 2010-09-14T00:00:00,MRF,136.0 592 | 2010-09-15T00:00:00,CZA,1025.0 593 | 2010-09-15T00:00:00,GBG,1735.0 594 | 2010-09-15T00:00:00,GDO,192.0 595 | 2010-09-16T00:00:00,CZA,1039.0 596 | 2010-09-16T00:00:00,MRF,137.0 597 | 2010-09-16T00:00:00,GBG,1775.0 598 | 2010-09-17T00:00:00,GDO,194.0 599 | 2010-09-17T00:00:00,MRF,142.0 600 | 2010-09-17T00:00:00,CZA,1050.0 601 | 2010-09-20T00:00:00,GBG,1755.0 602 | 2010-09-20T00:00:00,GDO,205.0 603 | 2010-09-20T00:00:00,MRF,149.0 604 | 2010-09-21T00:00:00,CZA,1031.0 605 | 2010-09-21T00:00:00,GBG,1772.0 606 | 2010-09-21T00:00:00,GDO,201.0 607 | 2010-09-21T00:00:00,MRF,148.0 608 | 2010-09-22T00:00:00,CZA,1019.0 609 | 2010-09-22T00:00:00,GBG,1780.0 610 | 2010-09-22T00:00:00,GDO,202.0 611 | 2010-09-22T00:00:00,MRF,144.0 612 | 2010-09-23T00:00:00,CZA,1015.0 613 | 2010-09-23T00:00:00,GBG,1741.0 614 | 2010-09-23T00:00:00,GDO,205.0 615 | 2010-09-27T00:00:00,CZA,994.0 616 | 2010-09-27T00:00:00,GBG,1680.0 617 | 2010-09-27T00:00:00,GDO,202.0 618 | 2010-09-27T00:00:00,MRF,146.0 619 | 2010-09-28T00:00:00,CZA,980.0 620 | 2010-09-28T00:00:00,GBG,1652.0 621 | 2010-09-28T00:00:00,GDO,206.0 622 | 2010-09-29T00:00:00,CZA,950.0 623 | 2010-09-29T00:00:00,GBG,1620.0 624 | 2010-09-29T00:00:00,GDO,205.0 625 | 2010-09-29T00:00:00,MRF,145.0 626 | 2010-09-30T00:00:00,CZA,955.0 627 | 2010-09-30T00:00:00,GBG,1719.0 628 | 2010-09-30T00:00:00,GDO,199.0 629 | 2010-10-01T00:00:00,CZA,857.0 630 | 2010-10-01T00:00:00,GBG,1725.0 631 | 2010-10-01T00:00:00,GDO,202.0 632 | 2010-10-01T00:00:00,MRF,144.0 633 | 2010-10-04T00:00:00,CZA,895.0 634 | 2010-10-04T00:00:00,GDO,197.0 635 | 2010-10-04T00:00:00,MRF,145.0 636 | 2010-10-04T00:00:00,GBG,1700.0 637 | 2010-10-05T00:00:00,CZA,866.0 638 | 2010-10-05T00:00:00,GDO,190.0 639 | 2010-10-05T00:00:00,MRF,144.0 640 | 2010-10-06T00:00:00,CZA,865.0 641 | 2010-10-06T00:00:00,GBG,1676.0 642 | 2010-10-06T00:00:00,GDO,202.0 643 | 2010-10-06T00:00:00,MRF,145.0 644 | 2010-10-07T00:00:00,CZA,904.0 645 | 2010-10-07T00:00:00,GBG,1674.0 646 | 2010-10-07T00:00:00,MRF,148.0 647 | 2010-10-07T00:00:00,GDO,210.0 648 | 2010-10-08T00:00:00,CZA,933.0 649 | 2010-10-08T00:00:00,GBG,1660.0 650 | 2010-10-08T00:00:00,MRF,151.0 651 | 2010-10-11T00:00:00,CZA,997.0 652 | 2010-10-11T00:00:00,GBG,1775.0 653 | 2010-10-11T00:00:00,GDO,229.0 654 | 2010-10-12T00:00:00,CZA,957.0 655 | 2010-10-12T00:00:00,GBG,1850.0 656 | 2010-10-12T00:00:00,GDO,235.0 657 | 2010-10-13T00:00:00,CZA,968.0 658 | 2010-10-13T00:00:00,GDO,258.0 659 | 2010-10-13T00:00:00,MRF,149.0 660 | 2010-10-13T00:00:00,GBG,1950.0 661 | 2010-10-14T00:00:00,CZA,1000.0 662 | 2010-10-14T00:00:00,GDO,249.0 663 | 2010-10-14T00:00:00,MRF,145.0 664 | 2010-10-15T00:00:00,CZA,1010.0 665 | 2010-10-15T00:00:00,GBG,1842.0 666 | 2010-10-15T00:00:00,GDO,237.0 667 | 2010-10-15T00:00:00,MRF,143.0 668 | 2010-10-18T00:00:00,CZA,983.0 669 | 2010-10-18T00:00:00,GBG,1840.0 670 | 2010-10-18T00:00:00,GDO,236.0 671 | 2010-10-18T00:00:00,MRF,141.0 672 | 2010-10-19T00:00:00,CZA,982.0 673 | 2010-10-19T00:00:00,GBG,1850.0 674 | 2010-10-19T00:00:00,GDO,248.0 675 | 2010-10-19T00:00:00,MRF,136.0 676 | 2010-10-20T00:00:00,CZA,985.0 677 | 2010-10-20T00:00:00,GDO,236.0 678 | 2010-10-20T00:00:00,MRF,135.0 679 | 2010-10-20T00:00:00,GBG,1780.0 680 | 2010-10-21T00:00:00,CZA,977.0 681 | 2010-10-21T00:00:00,GDO,245.0 682 | 2010-10-21T00:00:00,MRF,139.0 683 | 2010-10-22T00:00:00,CZA,967.0 684 | 2010-10-22T00:00:00,GBG,1704.0 685 | 2010-10-22T00:00:00,GDO,240.0 686 | 2010-10-22T00:00:00,MRF,137.0 687 | 2010-10-25T00:00:00,CZA,944.0 688 | 2010-10-25T00:00:00,GBG,1785.0 689 | 2010-10-25T00:00:00,GDO,244.0 690 | 2010-10-25T00:00:00,MRF,141.0 691 | 2010-10-26T00:00:00,CZA,950.0 692 | 2010-10-26T00:00:00,GBG,1853.0 693 | 2010-10-26T00:00:00,GDO,245.0 694 | 2010-10-26T00:00:00,MRF,139.0 695 | 2010-10-27T00:00:00,CZA,989.0 696 | 2010-10-27T00:00:00,GBG,1815.0 697 | 2010-10-27T00:00:00,GDO,240.0 698 | 2010-10-27T00:00:00,MRF,137.0 699 | 2010-10-28T00:00:00,CZA,975.0 700 | 2010-10-28T00:00:00,GBG,1850.0 701 | 2010-10-28T00:00:00,GDO,241.0 702 | 2010-10-29T00:00:00,CZA,971.0 703 | 2010-10-29T00:00:00,GBG,1900.0 704 | 2010-10-29T00:00:00,GDO,232.0 705 | 2010-10-29T00:00:00,MRF,138.0 706 | 2010-11-01T00:00:00,CZA,961.0 707 | 2010-11-01T00:00:00,GBG,1950.0 708 | 2010-11-01T00:00:00,GDO,229.0 709 | 2010-11-01T00:00:00,MRF,140.0 710 | 2010-11-02T00:00:00,CZA,954.0 711 | 2010-11-02T00:00:00,GBG,1940.0 712 | 2010-11-02T00:00:00,GDO,231.0 713 | 2010-11-02T00:00:00,MRF,139.0 714 | 2010-11-03T00:00:00,CZA,800.0 715 | 2010-11-03T00:00:00,GBG,1950.0 716 | 2010-11-03T00:00:00,GDO,238.0 717 | 2010-11-04T00:00:00,CZA,825.0 718 | 2010-11-04T00:00:00,GBG,1964.0 719 | 2010-11-04T00:00:00,GDO,237.0 720 | 2010-11-05T00:00:00,CZA,818.0 721 | 2010-11-05T00:00:00,GBG,2079.0 722 | 2010-11-05T00:00:00,GDO,240.0 723 | 2010-11-05T00:00:00,MRF,141.0 724 | 2010-11-08T00:00:00,CZA,814.0 725 | 2010-11-08T00:00:00,GBG,2015.0 726 | 2010-11-08T00:00:00,GDO,244.0 727 | 2010-11-08T00:00:00,MRF,138.0 728 | 2010-11-08T00:00:00,RBP,6490.0 729 | 2010-11-09T00:00:00,CZA,800.0 730 | 2010-11-09T00:00:00,GBG,2171.0 731 | 2010-11-09T00:00:00,GDO,253.0 732 | 2010-11-09T00:00:00,MRF,143.0 733 | 2010-11-09T00:00:00,RBP,6880.0 734 | 2010-11-10T00:00:00,CZA,769.0 735 | 2010-11-10T00:00:00,GDO,252.0 736 | 2010-11-10T00:00:00,MRF,157.0 737 | 2010-11-10T00:00:00,RBP,6885.0 738 | 2010-11-10T00:00:00,GBG,2123.0 739 | 2010-11-11T00:00:00,CZA,784.0 740 | 2010-11-11T00:00:00,GDO,258.0 741 | 2010-11-11T00:00:00,MRF,158.0 742 | 2010-11-11T00:00:00,RBP,6895.0 743 | 2010-11-12T00:00:00,CZA,870.0 744 | 2010-11-12T00:00:00,GBG,2108.0 745 | 2010-11-12T00:00:00,GDO,264.0 746 | 2010-11-12T00:00:00,MRF,155.0 747 | 2010-11-12T00:00:00,RBP,6820.0 748 | 2010-11-15T00:00:00,CZA,885.0 749 | 2010-11-15T00:00:00,GBG,2100.0 750 | 2010-11-15T00:00:00,GDO,260.0 751 | 2010-11-15T00:00:00,RBP,6835.0 752 | 2010-11-15T00:00:00,MRF,156.0 753 | 2010-11-16T00:00:00,CZA,860.0 754 | 2010-11-16T00:00:00,GBG,1965.0 755 | 2010-11-16T00:00:00,RBP,6700.0 756 | 2010-11-16T00:00:00,GDO,255.0 757 | 2010-11-17T00:00:00,CZA,842.0 758 | 2010-11-17T00:00:00,GBG,1881.0 759 | 2010-11-17T00:00:00,MRF,157.0 760 | 2010-11-17T00:00:00,RBP,6701.0 761 | 2010-11-18T00:00:00,CZA,874.0 762 | 2010-11-18T00:00:00,GBG,1955.0 763 | 2010-11-18T00:00:00,GDO,254.0 764 | 2010-11-18T00:00:00,MRF,163.0 765 | 2010-11-18T00:00:00,RBP,6699.0 766 | 2010-11-19T00:00:00,CZA,861.0 767 | 2010-11-19T00:00:00,GBG,1925.0 768 | 2010-11-19T00:00:00,MRF,166.0 769 | 2010-11-19T00:00:00,RBP,6720.0 770 | 2010-11-19T00:00:00,GDO,250.0 771 | 2010-11-22T00:00:00,CZA,850.0 772 | 2010-11-22T00:00:00,MRF,160.0 773 | 2010-11-22T00:00:00,RBP,6676.0 774 | 2010-11-22T00:00:00,GBG,1937.0 775 | 2010-11-23T00:00:00,CZA,837.0 776 | 2010-11-23T00:00:00,GDO,244.0 777 | 2010-11-23T00:00:00,RBP,6450.0 778 | 2010-11-23T00:00:00,MRF,158.0 779 | 2010-11-24T00:00:00,CZA,848.0 780 | 2010-11-24T00:00:00,GDO,247.0 781 | 2010-11-24T00:00:00,RBP,6475.0 782 | 2010-11-25T00:00:00,CZA,840.0 783 | 2010-11-25T00:00:00,GDO,243.0 784 | 2010-11-25T00:00:00,MRF,161.0 785 | 2010-11-25T00:00:00,RBP,6490.0 786 | 2010-11-25T00:00:00,GBG,1840.0 787 | 2010-11-26T00:00:00,CZA,820.0 788 | 2010-11-26T00:00:00,GDO,250.0 789 | 2010-11-26T00:00:00,MRF,160.0 790 | 2010-11-26T00:00:00,RBP,6450.0 791 | 2010-11-29T00:00:00,CZA,840.0 792 | 2010-11-29T00:00:00,GBG,1946.0 793 | 2010-11-29T00:00:00,GDO,239.0 794 | 2010-11-29T00:00:00,MRF,155.0 795 | 2010-11-29T00:00:00,RBP,6252.0 796 | 2010-11-30T00:00:00,CZA,820.0 797 | 2010-11-30T00:00:00,GBG,2000.0 798 | 2010-11-30T00:00:00,GDO,244.0 799 | 2010-11-30T00:00:00,MRF,150.0 800 | 2010-11-30T00:00:00,RBP,6500.0 801 | 2010-12-01T00:00:00,CZA,814.0 802 | 2010-12-01T00:00:00,GBG,2073.0 803 | 2010-12-01T00:00:00,GDO,249.0 804 | 2010-12-01T00:00:00,MRF,155.0 805 | 2010-12-01T00:00:00,RBP,6550.0 806 | 2010-12-02T00:00:00,CZA,792.0 807 | 2010-12-02T00:00:00,GBG,2009.0 808 | 2010-12-02T00:00:00,GDO,250.0 809 | 2010-12-02T00:00:00,MRF,151.0 810 | 2010-12-02T00:00:00,RBP,6700.0 811 | 2010-12-03T00:00:00,CZA,821.0 812 | 2010-12-03T00:00:00,GBG,2004.0 813 | 2010-12-03T00:00:00,GDO,239.0 814 | 2010-12-03T00:00:00,RBP,6520.0 815 | 2010-12-03T00:00:00,MRF,153.0 816 | 2010-12-06T00:00:00,CZA,854.0 817 | 2010-12-06T00:00:00,GBG,2025.0 818 | 2010-12-06T00:00:00,GDO,245.0 819 | 2010-12-06T00:00:00,RBP,6595.0 820 | 2010-12-07T00:00:00,GBG,1966.0 821 | 2010-12-07T00:00:00,MRF,155.0 822 | 2010-12-07T00:00:00,RBP,6500.0 823 | 2010-12-07T00:00:00,CZA,915.0 824 | 2010-12-07T00:00:00,GDO,250.0 825 | 2010-12-08T00:00:00,MRF,157.0 826 | 2010-12-08T00:00:00,RBP,6525.0 827 | 2010-12-08T00:00:00,GBG,1950.0 828 | 2010-12-09T00:00:00,CZA,920.0 829 | 2010-12-09T00:00:00,GDO,255.0 830 | 2010-12-09T00:00:00,RBP,6485.0 831 | 2010-12-09T00:00:00,MRF,159.0 832 | 2010-12-10T00:00:00,CZA,902.0 833 | 2010-12-10T00:00:00,GBG,1856.0 834 | 2010-12-10T00:00:00,GDO,245.0 835 | 2010-12-10T00:00:00,RBP,6480.0 836 | 2010-12-13T00:00:00,CZA,901.0 837 | 2010-12-13T00:00:00,GBG,1850.0 838 | 2010-12-13T00:00:00,GDO,244.0 839 | 2010-12-13T00:00:00,MRF,161.0 840 | 2010-12-13T00:00:00,RBP,6500.0 841 | 2010-12-14T00:00:00,CZA,900.0 842 | 2010-12-14T00:00:00,GBG,1865.0 843 | 2010-12-14T00:00:00,GDO,250.0 844 | 2010-12-14T00:00:00,MRF,155.0 845 | 2010-12-15T00:00:00,CZA,885.0 846 | 2010-12-15T00:00:00,GBG,1860.0 847 | 2010-12-15T00:00:00,GDO,238.0 848 | 2010-12-15T00:00:00,MRF,157.0 849 | 2010-12-15T00:00:00,RBP,6595.0 850 | 2010-12-17T00:00:00,CZA,880.0 851 | 2010-12-17T00:00:00,GBG,1940.0 852 | 2010-12-17T00:00:00,GDO,234.0 853 | 2010-12-17T00:00:00,RBP,6491.0 854 | 2010-12-17T00:00:00,MRF,155.0 855 | 2010-12-20T00:00:00,CZA,920.0 856 | 2010-12-20T00:00:00,GBG,2000.0 857 | 2010-12-20T00:00:00,GDO,230.0 858 | 2010-12-20T00:00:00,RBP,6520.0 859 | 2010-12-21T00:00:00,CZA,935.0 860 | 2010-12-21T00:00:00,GBG,1972.0 861 | 2010-12-21T00:00:00,GDO,238.0 862 | 2010-12-21T00:00:00,MRF,157.0 863 | 2010-12-21T00:00:00,RBP,6598.0 864 | 2010-12-22T00:00:00,CZA,930.0 865 | 2010-12-22T00:00:00,GBG,2000.0 866 | 2010-12-22T00:00:00,GDO,229.0 867 | 2010-12-22T00:00:00,MRF,158.0 868 | 2010-12-22T00:00:00,RBP,6580.0 869 | 2010-12-23T00:00:00,MRF,159.0 870 | 2010-12-23T00:00:00,CZA,950.0 871 | 2010-12-23T00:00:00,GBG,1950.0 872 | 2010-12-23T00:00:00,GDO,225.0 873 | 2010-12-23T00:00:00,RBP,6700.0 874 | 2010-12-24T00:00:00,MRF,160.0 875 | 2010-12-28T00:00:00,CZA,951.0 876 | 2010-12-28T00:00:00,GBG,2250.0 877 | 2010-12-28T00:00:00,MRF,158.0 878 | 2010-12-29T00:00:00,CZA,900.0 879 | 2010-12-29T00:00:00,GBG,2199.0 880 | 2010-12-29T00:00:00,GDO,238.0 881 | 2010-12-30T00:00:00,CZA,935.0 882 | 2010-12-30T00:00:00,GBG,1925.0 883 | 2010-12-30T00:00:00,MRF,160.0 884 | 2010-12-30T00:00:00,RBP,6698.0 885 | 2010-12-30T00:00:00,GDO,240.0 886 | 2010-12-31T00:00:00,CZA,958.0 887 | 2010-12-31T00:00:00,GBG,1980.0 888 | 2010-12-31T00:00:00,MRF,166.0 889 | 2010-12-31T00:00:00,RBP,6675.0 890 | 2010-12-31T00:00:00,GDO,240.0 -------------------------------------------------------------------------------- /fn_graph/examples/shares_in_issue.csv: -------------------------------------------------------------------------------- 1 | datetime,share_code,shares_in_issue 2 | 2010-01-04T00:00:00,GDO,804966976.0 3 | 2010-01-04T00:00:00,CZA,474414016.0 4 | 2010-01-04T00:00:00,GBG,334004992.0 5 | 2010-01-04T00:00:00,MRF,2459259904.0 6 | 2010-01-11T00:00:00,GDO,805187008.0 7 | 2010-01-21T00:00:00,CZA,474539008.0 8 | 2010-01-22T00:00:00,GBG,334158016.0 9 | 2010-01-25T00:00:00,GDO,805238976.0 10 | 2010-02-08T00:00:00,GBG,334743008.0 11 | 2010-03-10T00:00:00,CZA,480515008.0 12 | 2010-03-11T00:00:00,GDO,805240000.0 13 | 2010-03-18T00:00:00,MRF,2459340032.0 14 | 2010-03-31T00:00:00,GBG,337816992.0 15 | 2010-04-09T00:00:00,GBG,337975008.0 16 | 2010-04-16T00:00:00,MRF,2460339968.0 17 | 2010-04-21T00:00:00,MRF,2460509952.0 18 | 2010-04-26T00:00:00,GBG,338121984.0 19 | 2010-05-06T00:00:00,GDO,805448000.0 20 | 2010-05-17T00:00:00,GDO,805894976.0 21 | 2010-05-19T00:00:00,GBG,338153984.0 22 | 2010-05-31T00:00:00,GBG,340388000.0 23 | 2010-06-22T00:00:00,GBG,340440992.0 24 | 2010-06-23T00:00:00,CZA,530515008.0 25 | 2010-07-09T00:00:00,GDO,806163008.0 26 | 2010-07-14T00:00:00,GDO,806268032.0 27 | 2010-07-15T00:00:00,GBG,340441984.0 28 | 2010-07-21T00:00:00,GBG,340504992.0 29 | 2010-08-31T00:00:00,GBG,348004992.0 30 | 2010-09-08T00:00:00,MRF,2461839872.0 31 | 2010-09-09T00:00:00,GBG,348263008.0 32 | 2010-09-15T00:00:00,GBG,352777984.0 33 | 2010-09-17T00:00:00,GBG,353248992.0 34 | 2010-09-27T00:00:00,MRF,2463340032.0 35 | 2010-10-19T00:00:00,GBG,367969984.0 36 | 2010-10-25T00:00:00,GBG,368260992.0 37 | 2010-10-27T00:00:00,GDO,806307968.0 38 | 2010-10-29T00:00:00,MRF,2472529920.0 39 | 2010-11-08T00:00:00,MRF,2476829952.0 40 | 2010-11-08T00:00:00,RBP,164095008.0 41 | 2010-11-18T00:00:00,GBG,406452000.0 42 | 2010-11-24T00:00:00,GBG,408366016.0 43 | 2010-11-25T00:00:00,MRF,2475650048.0 44 | 2010-11-26T00:00:00,GDO,806876032.0 45 | 2010-12-02T00:00:00,MRF,2476659968.0 46 | 2010-12-15T00:00:00,GBG,409835008.0 47 | 2010-12-20T00:00:00,GBG,411910016.0 48 | -------------------------------------------------------------------------------- /fn_graph/examples/stock_market.py: -------------------------------------------------------------------------------- 1 | """ 2 | A simple example that works out the market capitalization fo a couple of stocks. 3 | """ 4 | # %% 5 | from fn_graph import Composer 6 | from pathlib import Path 7 | import pandas as pd 8 | import numpy as np 9 | import plotly.express as px 10 | 11 | data_path = Path(__file__).parent 12 | 13 | 14 | def share_prices(): 15 | """ 16 | Load the share price data 17 | """ 18 | return pd.read_csv(data_path / "share_prices.csv", parse_dates=["datetime"]) 19 | 20 | 21 | def shares_in_issue(): 22 | """ 23 | Load the shares issued data 24 | """ 25 | return pd.read_csv(data_path / "shares_in_issue.csv", parse_dates=["datetime"]) 26 | 27 | 28 | def daily_share_prices(share_prices): 29 | """ 30 | Ensure that every day has the full set of share prices 31 | """ 32 | return ( 33 | share_prices.groupby("share_code") 34 | .apply(lambda df: df.set_index("datetime").resample("1D").ffill().reset_index()) 35 | .reset_index(drop=True) 36 | .sort_values(by=["datetime", "share_code"]) 37 | ) 38 | 39 | 40 | def market_cap(daily_share_prices, shares_in_issue): 41 | """ 42 | Merge the datasets intelligently over time and calculate market cap 43 | """ 44 | return pd.merge_asof( 45 | daily_share_prices, shares_in_issue, on="datetime", by="share_code" 46 | ).assign(market_cap=lambda df: df.share_price * df.shares_in_issue) 47 | 48 | 49 | def total_market_cap(market_cap): 50 | """ 51 | Workout the total market cap 52 | """ 53 | return market_cap.groupby("datetime", as_index=False).market_cap.sum() 54 | 55 | 56 | def total_market_cap_change(total_market_cap, swing_threshold): 57 | """ 58 | Calculate the changes in market cap 59 | """ 60 | return total_market_cap.assign( 61 | market_cap_change=lambda df: df.market_cap.diff() 62 | ).assign( 63 | change_classification=lambda df: np.where( 64 | np.abs(df.market_cap_change) > swing_threshold, "large", "small" 65 | ) 66 | ) 67 | 68 | 69 | def plot_market_caps(market_cap): 70 | """ 71 | Plot the individual market caps 72 | """ 73 | return px.area( 74 | market_cap, 75 | x="datetime", 76 | y="market_cap", 77 | facet_row="share_code", 78 | color="share_code", 79 | ) 80 | 81 | 82 | def plot_total_market_cap(total_market_cap): 83 | """ 84 | Plot the total market cap 85 | """ 86 | return px.line(total_market_cap, x="datetime", y="market_cap") 87 | 88 | 89 | def plot_market_cap_changes(total_market_cap_change): 90 | """ 91 | Plot the market cap changes 92 | """ 93 | return px.bar( 94 | total_market_cap_change, 95 | x="datetime", 96 | y="market_cap_change", 97 | color="change_classification", 98 | ) 99 | 100 | 101 | f = ( 102 | Composer() 103 | .update_parameters(swing_threshold=0.1 * 10 ** 12) 104 | .update( 105 | share_prices, 106 | daily_share_prices, 107 | shares_in_issue, 108 | market_cap, 109 | total_market_cap, 110 | total_market_cap_change, 111 | plot_market_caps, 112 | plot_total_market_cap, 113 | plot_market_cap_changes, 114 | ) 115 | ) 116 | -------------------------------------------------------------------------------- /fn_graph/profiler.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | 4 | def now(): 5 | return time.perf_counter() 6 | 7 | 8 | class Profiler: 9 | def __init__(self): 10 | self.start = {} 11 | self.end = {} 12 | 13 | def __call__(self, event_type, details): 14 | name = details.get("name") 15 | instruction = { 16 | "start_calculation": ("start", "calculation", "preparation"), 17 | "prepared_calculation": ("end", "calculation", "preparation"), 18 | "start_step": ("start", "step", name), 19 | "end_step": ("end", "step", name), 20 | "start_function": ("start", "execution", name), 21 | "end_function": ("end", "execution", name), 22 | "start_cache_retrieval": ("start", "cache_retrieval", name), 23 | "end_cache_retrieval": ("end", "cache_retrieval", name), 24 | "start_cache_store": ("start", "cache_store", name), 25 | "end_cache_store": ("end", "cache_store", name), 26 | }.get(event_type) 27 | 28 | if instruction: 29 | event, category, name = instruction 30 | store = getattr(self, event) 31 | store[(category, name)] = now() 32 | 33 | def results(self): 34 | def period(key): 35 | return self.end.get(key, 0) - self.start.get(key, 0) 36 | 37 | def function_profile(name): 38 | total = period(("step", name)) 39 | execution = period(("execution", name)) 40 | retrieval = period(("cache_retrieval", name)) 41 | store = period(("cache_store", name)) 42 | 43 | overhead = total - execution - retrieval - store 44 | return dict( 45 | total=total, 46 | overhead=overhead, 47 | execution=execution, 48 | cache_retrieval=retrieval, 49 | cache_store=store, 50 | ) 51 | 52 | names = [name for event, name in self.start.keys() if event == "step"] 53 | 54 | return dict( 55 | startup=dict( 56 | preparation=dict( 57 | preparation=period(("calculation", "preparation")), 58 | total=period(("calculation", "preparation")), 59 | ) 60 | ), 61 | functions={name: function_profile(name) for name in names}, 62 | ) 63 | -------------------------------------------------------------------------------- /fn_graph/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BusinessOptics/fn_graph/50ef05ce07c069383850cbdba190af790278419b/fn_graph/tests/__init__.py -------------------------------------------------------------------------------- /fn_graph/tests/large_graph.py: -------------------------------------------------------------------------------- 1 | from fn_graph import Composer 2 | 3 | 4 | def function_0(): 5 | return 42 6 | 7 | 8 | def function_1(): 9 | return 43 10 | 11 | 12 | def function_2(function_1): 13 | return function_1 14 | 15 | 16 | def function_3(function_1): 17 | return function_1 18 | 19 | 20 | def function_4(function_0): 21 | return function_0 22 | 23 | 24 | def function_5(function_2): 25 | return function_2 26 | 27 | 28 | def function_6(function_0, function_2, function_3, function_5): 29 | return function_3 + function_2 + function_5 + function_0 30 | 31 | 32 | def function_7(function_2, function_3, function_5, function_6): 33 | return function_3 + function_2 + function_6 + function_5 34 | 35 | 36 | # making these links 37 | 38 | # def function_8(function_7): 39 | # return function_7 40 | 41 | 42 | # def function_9(function_5): 43 | # return function_5 44 | 45 | 46 | def function_10(function_1, function_3, function_6, function_7, function_9): 47 | return function_3 + function_1 + function_6 + function_7 + function_9 48 | 49 | 50 | def function_11(function_1, function_4): 51 | return function_4 + function_1 52 | 53 | 54 | def function_12(function_3, function_5, function_6, function_8, function_9): 55 | return function_3 + function_6 + function_8 + function_9 + function_5 56 | 57 | 58 | def function_13(function_0, function_12, function_4, function_8): 59 | return function_12 + function_4 + function_8 + function_0 60 | 61 | 62 | def function_14(function_7, function_9): 63 | return function_7 + function_9 64 | 65 | 66 | def function_15(function_0, function_14, function_3, function_7): 67 | return function_3 + function_7 + function_14 + function_0 68 | 69 | 70 | def function_16(function_0, function_1, function_11, function_3, function_6): 71 | return function_11 + function_3 + function_1 + function_6 + function_0 72 | 73 | 74 | def function_17(function_2, function_8): 75 | return function_2 + function_8 76 | 77 | 78 | def function_18(function_10, function_13, function_14, function_4): 79 | return function_4 + function_10 + function_14 + function_13 80 | 81 | 82 | def function_19(function_10, function_7): 83 | return function_10 + function_7 84 | 85 | 86 | def function_20(function_15): 87 | return function_15 88 | 89 | 90 | def function_21(function_4): 91 | return function_4 92 | 93 | 94 | def function_22(function_10, function_15, function_9): 95 | return function_15 + function_10 + function_9 96 | 97 | 98 | def function_23(function_15, function_6): 99 | return function_15 + function_6 100 | 101 | 102 | def function_24(function_1, function_12, function_18, function_23, function_5): 103 | return function_12 + function_1 + function_18 + function_23 + function_5 104 | 105 | 106 | def function_25(function_14, function_18): 107 | return function_18 + function_14 108 | 109 | 110 | def function_26(function_0, function_14, function_2): 111 | return function_2 + function_14 + function_0 112 | 113 | 114 | def function_27(function_26): 115 | return function_26 116 | 117 | 118 | def function_28(function_17, function_2, function_24, function_25, function_9): 119 | return function_17 + function_25 + function_24 + function_2 + function_9 120 | 121 | 122 | def function_29(): 123 | return 29 124 | 125 | 126 | def function_30(function_13, function_25): 127 | return function_25 + function_13 128 | 129 | 130 | def function_31(function_1, function_14, function_15, function_29, function_7): 131 | return function_15 + function_1 + function_29 + function_7 + function_14 132 | 133 | 134 | def function_32(function_12, function_22, function_25): 135 | return function_12 + function_22 + function_25 136 | 137 | 138 | def function_33(function_3): 139 | return function_3 140 | 141 | 142 | def function_34(function_25, function_33, function_4, function_5, function_6): 143 | return function_4 + function_25 + function_6 + function_33 + function_5 144 | 145 | 146 | def function_35(function_14, function_17, function_21): 147 | return function_17 + function_21 + function_14 148 | 149 | 150 | def function_36(function_33, function_6): 151 | return function_33 + function_6 152 | 153 | 154 | def function_37(): 155 | return 19 156 | 157 | 158 | def function_38(function_33, function_34): 159 | return function_34 + function_33 160 | 161 | 162 | def function_39(function_1, function_10, function_25): 163 | return function_25 + function_1 + function_10 164 | 165 | 166 | def function_40(function_21, function_24, function_33): 167 | return function_21 + function_33 + function_24 168 | 169 | 170 | def function_41(function_18): 171 | return function_18 172 | 173 | 174 | #%% 175 | 176 | scope = locals() 177 | functions = [scope[f"function_{i}"] for i in range(42) if f"function_{i}" in scope] 178 | large_graph = ( 179 | Composer().update(*functions).link(function_8="function_7", function_9="function_5") 180 | ) 181 | -------------------------------------------------------------------------------- /fn_graph/tests/test_basics.py: -------------------------------------------------------------------------------- 1 | from fn_graph import Composer 2 | 3 | 4 | def test_simple_parameters(): 5 | composer = ( 6 | Composer().update(c=lambda a, b: a + b).update_parameters(a=1, b=(int, 2)) 7 | ) 8 | assert composer.c() == 3 9 | 10 | 11 | def test_default_arguments(): 12 | composer = Composer().update(c=lambda a, b=3: a + b).update_parameters(a=1) 13 | assert composer.c() == 4 14 | composer = Composer().update(c=lambda a, b=3: a + b).update_parameters(a=1, b=2) 15 | assert composer.c() == 3 16 | 17 | 18 | def test_var_args(): 19 | a = 1 20 | b = 2 21 | result = a + sum(a * 2 for i in range(10)) + b + sum(a * 5 for i in range(5)) 22 | 23 | def d(a, *d_, b, **c_): 24 | return a + sum(d_) + b + sum(c_.values()) 25 | 26 | composer = ( 27 | Composer() 28 | .update_parameters(a=1, b=2) 29 | .update(**{f"c_{i}": lambda a: a * 2 for i in range(10)}) 30 | .update(**{f"d_{i}": lambda a: a * 5 for i in range(5)}) 31 | .update(d) 32 | ) 33 | 34 | assert composer.d() == result 35 | 36 | 37 | def test_empty_var_args(): 38 | a = 1 39 | b = 2 40 | result = a + sum(a * 2 for i in range(10)) + b 41 | 42 | def d(a, *d_, b, **c_): 43 | return a + sum(d_) + b + sum(c_.values()) 44 | 45 | composer = ( 46 | Composer() 47 | .update_parameters(a=1, b=2) 48 | .update(**{f"c_{i}": lambda a: a * 2 for i in range(10)}) 49 | .update(d) 50 | ) 51 | 52 | assert composer.d() == result 53 | 54 | 55 | def test_empty_kwargs(): 56 | a = 1 57 | b = 2 58 | result = a + b + sum(a * 5 for i in range(5)) 59 | 60 | def d(a, *d_, b, **c_): 61 | return a + sum(d_) + b + sum(c_.values()) 62 | 63 | composer = ( 64 | Composer() 65 | .update_parameters(a=1, b=2) 66 | .update(**{f"d_{i}": lambda a: a * 5 for i in range(5)}) 67 | .update(d) 68 | ) 69 | 70 | assert composer.d() == result 71 | -------------------------------------------------------------------------------- /fn_graph/tests/test_cache_methods.py: -------------------------------------------------------------------------------- 1 | from .utils import compare_composer_results, generate_random_graph 2 | from random import choice 3 | from fn_graph.tests.large_graph import large_graph 4 | from fn_graph import Composer 5 | from ..calculation import get_execution_instructions 6 | 7 | root = large_graph 8 | composers = [root, root.cache(), root.development_cache(__name__)] 9 | 10 | 11 | def test_static_cache_equality(): 12 | root = large_graph 13 | composers = [root, root.cache(), root.development_cache(__name__)] 14 | compare_composer_results(root, composers) 15 | 16 | 17 | def test_cache_clear(): 18 | for i in range(5): 19 | for composer in composers: 20 | composer.call(choice(list(composer.dag().nodes()))) 21 | composer.cache_clear() 22 | 23 | 24 | def test_cache_graphviz(): 25 | for composer in composers: 26 | composer.call(choice(list(composer.dag().nodes()))) 27 | composer.cache_invalidate(choice(list(composer.dag().nodes()))) 28 | composer.cache_graphviz() 29 | 30 | 31 | def test_same_graphs(): 32 | for i in range(10): 33 | root = generate_random_graph() 34 | composers = [root, root, root] 35 | compare_composer_results(root, composers) 36 | 37 | 38 | def test_random_graphs(): 39 | for i in range(10): 40 | root = generate_random_graph() 41 | composers = [root, root.cache()] 42 | compare_composer_results(root, composers) 43 | 44 | 45 | def test_cache_invalidation(): 46 | composer = Composer().update_parameters(a=1).cache() 47 | assert composer.a() == 1 48 | composer = composer.update_parameters(a=2) 49 | assert composer.a() == 2 50 | 51 | 52 | def test_shared_cache(): 53 | c = Composer().update(foo=lambda x: x * 2, bar=lambda: 9).cache() 54 | assert c.bar() == 9 55 | assert c.update_parameters(x=2).foo() == 4 56 | assert c.bar() == 9 # KeyError: 'x' 57 | assert c.update_parameters(x=2).foo() == 4 58 | -------------------------------------------------------------------------------- /fn_graph/tests/test_links.py: -------------------------------------------------------------------------------- 1 | from fn_graph import Composer 2 | 3 | 4 | def test_basic_links(): 5 | composer = Composer().update(a=lambda: 5, c=lambda b: b * 2).link(b="a") 6 | 7 | assert composer.c() == 10 8 | 9 | 10 | def test_updated_from_links(): 11 | composer_a = Composer().update(a=lambda: 5, c=lambda b: b * 2).link(b="a") 12 | composer_b = Composer().update_from(composer_a).update(d=lambda b: b * 3) 13 | assert composer_b.d() == 15 14 | 15 | 16 | def test_updated_from_namespaces(): 17 | composer_child = Composer().update(a=lambda: 5, c=lambda b: b * 2).link(b="a") 18 | composer = ( 19 | Composer() 20 | .update_namespaces(x=composer_child, y=composer_child) 21 | .link(outer_x="x__c", outer_y="y__c") 22 | .update(final=lambda outer_x, outer_y: outer_x + outer_y) 23 | ) 24 | 25 | assert composer.final() == 20 26 | 27 | 28 | def test_call_link(): 29 | composer = Composer().update(a=lambda: 5, c=lambda b: b * 2).link(b="a") 30 | 31 | assert composer.b() == 5 32 | -------------------------------------------------------------------------------- /fn_graph/tests/utils.py: -------------------------------------------------------------------------------- 1 | from random import randint, choice 2 | from fn_graph import Composer 3 | from textwrap import dedent 4 | 5 | 6 | def check_all_equal(values): 7 | return any([a == b for a, b in list(zip(values[:-1], values[1:]))]) 8 | 9 | 10 | def check_results_equal(results): 11 | if not check_all_equal([set(result.keys()) for result in results]): 12 | raise Exception(f"Keys differ") 13 | 14 | for key in results[0]: 15 | result_values = [result[key] for result in results] 16 | if not check_all_equal(result_values): 17 | raise Exception(f"Difference found in {key}, results: {result_values}") 18 | 19 | 20 | def compare_composer_results(root, composers): 21 | nodes = root.dag().nodes() 22 | for j in range(10): 23 | outputs = set((choice(list(nodes)) for k in range(randint(1, len(nodes))))) 24 | intermediates = randint(0, 1) == 1 25 | results = [ 26 | composer.calculate(outputs, intermediates=intermediates) 27 | for composer in composers 28 | ] 29 | check_results_equal(results) 30 | 31 | for composer in composers: 32 | composer.cache_invalidate(choice(list(nodes))) 33 | 34 | 35 | def generate_random_graph(graph_size=42): 36 | 37 | functions = [] 38 | 39 | def function_0(): 40 | return 42 41 | 42 | functions.append(function_0) 43 | 44 | for i in range(1, graph_size): 45 | fn_name = f"function_{i}" 46 | num_args = randint(0, min(5, i - 1)) 47 | arg_names = set() 48 | 49 | while len(arg_names) < num_args: 50 | arg_name = f"function_{randint(0, i-1)}" 51 | if arg_name not in arg_names: 52 | arg_names.add(arg_name) 53 | 54 | if arg_names: 55 | body = " + ".join(arg_names) 56 | else: 57 | body = str(randint(0, 100)) 58 | 59 | exec( 60 | dedent( 61 | f""" 62 | def {fn_name}({', '.join(sorted(arg_names))}): 63 | return {body} 64 | """ 65 | ) 66 | ) 67 | functions.append(locals()[fn_name]) 68 | 69 | return Composer().update(*functions) 70 | -------------------------------------------------------------------------------- /fn_graph/usage.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from pathlib import Path 3 | import random 4 | from random import choice, random 5 | import pandas as pd 6 | 7 | sys.path.insert(0, str(Path(__file__).parent.parent.parent.resolve())) 8 | 9 | from fn_graph import Composer 10 | 11 | # Introductory example 12 | 13 | 14 | def a(): 15 | return 5 16 | 17 | 18 | def b(a): 19 | return a * 5 20 | 21 | 22 | def c(a, b): 23 | return a * b 24 | 25 | 26 | composer = Composer().update(a, b, c) 27 | 28 | # Call any result 29 | composer.c() # 125 30 | composer.a() # 5 31 | 32 | composer.graphviz().render("intro.gv", format="png") 33 | # Some pure functions 34 | 35 | 36 | def get_car_prices(): 37 | df = pd.DataFrame( 38 | dict( 39 | model=[choice(["corolla", "beetle", "ferrari"]) for _ in range(10)], 40 | price=[random() * 100_000 + 50000 for _ in range(10)], 41 | ) 42 | ) 43 | return df 44 | 45 | 46 | def get_mean_car_price(car_prices, season="summer"): 47 | if season != "summer": 48 | return car_prices.price.mean() / 2 49 | else: 50 | return car_prices.price.mean() 51 | 52 | 53 | def get_cheaper_cars(car_prices, your_car_price): 54 | df = car_prices 55 | return df[df.price < your_car_price] 56 | 57 | 58 | def get_savings_on_cheaper_cars(cheaper_cars, mean_car_price): 59 | return cheaper_cars.assign(savings=lambda df: mean_car_price - df.price) 60 | 61 | 62 | # Compose the functions 63 | g = ( 64 | Composer() 65 | .update_without_prefix( 66 | "get_", 67 | get_car_prices, 68 | get_cheaper_cars, 69 | get_mean_car_price, 70 | get_savings_on_cheaper_cars, 71 | ) 72 | .update_parameters(your_car_price=100_000) 73 | ) 74 | # Calculate a function 75 | print(g.calculate(["savings_on_cheaper_cars"])) 76 | 77 | # Or many functions 78 | print(g.calculate(["savings_on_cheaper_cars", "mean_car_price"])) 79 | 80 | # Or use the shorthand 81 | 82 | print(g.savings_on_cheaper_cars()) 83 | 84 | # Override a default argument 85 | g.update(season=lambda: "spring").cheaper_cars() 86 | print(g.savings_on_cheaper_cars()) 87 | 88 | # Lets create some more functions and create a partial composition 89 | 90 | 91 | def burger_savings(savings_df, price_of_a_burger): 92 | return savings_df.assign(burgers_saved=lambda df: df.savings / price_of_a_burger) 93 | 94 | 95 | f = Composer().update(price_of_a_burger=lambda: 100).update(burger_savings) 96 | 97 | # We can see we have some errors 98 | print(list(f.check())) 99 | 100 | 101 | # Lets compose g and f together 102 | # We also add another function to bridge the name 103 | 104 | h = g.update_from(f).update( 105 | savings_df=lambda savings_on_cheaper_cars: savings_on_cheaper_cars 106 | ) 107 | 108 | # see no errors 109 | print(list(h.check())) 110 | 111 | 112 | # Lets calculate the result while logging the progress 113 | def log_progress(event, details): 114 | time = f'{details.get("time"): .5f}' if "time" in details else "" 115 | print(f"{event:20s} {details.get('fname','')[:50]:50} {time}") 116 | 117 | 118 | print( 119 | h.update(season=lambda: "winter").calculate( 120 | ["burger_savings"], progress_callback=log_progress 121 | )["burger_savings"] 122 | ) 123 | 124 | 125 | # You can't have cycles 126 | cycles = Composer().update(a=lambda c: c, b=lambda a: a, c=lambda b: b) 127 | print(list(cycles.check())) 128 | 129 | 130 | # Lets create a new composer using namespaces 131 | # We also link the ideas that don't have matching names 132 | # We use "__" to separate namespace components 133 | h = ( 134 | Composer() 135 | .update_namespaces(cars=g, burgers=f) 136 | .link(burgers__savings_df="cars__savings_on_cheaper_cars") 137 | ) 138 | 139 | 140 | # There is a lot you can do with name spaces. 141 | h.calculate(["burgers__burger_savings"]) 142 | 143 | # If you draw the graph you can see the namespaces 144 | h.graphviz() 145 | 146 | g.calculate(["savings_on_cheaper_cars"], intermediates=False) 147 | 148 | # %% 149 | -------------------------------------------------------------------------------- /intro.gv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BusinessOptics/fn_graph/50ef05ce07c069383850cbdba190af790278419b/intro.gv.png -------------------------------------------------------------------------------- /linked_namespaces.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BusinessOptics/fn_graph/50ef05ce07c069383850cbdba190af790278419b/linked_namespaces.png -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "fn_graph" 3 | version = "0.14.3" 4 | description = "Manage, maintain and reuse complex function graphs without the hassle." 5 | authors = ["James Saunders "] 6 | license = "MIT" 7 | readme = "README.md" 8 | documentation = "https://fn-graph.readthedocs.io/" 9 | repository = "https://github.com/BusinessOptics/fn_graph" 10 | homepage = "https://github.com/BusinessOptics/fn_graph" 11 | 12 | [tool.poetry.dependencies] 13 | python = "^3.7" 14 | networkx = "^2.4" 15 | graphviz = "^0.13.2" 16 | littleutils = "^0.2.1" 17 | # below `extras`. They can be opted into for examples. 18 | seaborn = { version = "*", optional = true } 19 | statsmodels = { version = "*", optional = true } 20 | matplotlib = { version = "*", optional = true } 21 | sklearn = { version = "*", optional = true } 22 | plotly = { version = "*", optional = true } 23 | pandas = { version = "*", optional = true } 24 | yfinance = { version = "*", optional = true } 25 | 26 | [tool.poetry.dev-dependencies] 27 | black = { version = "^18.3-alpha.0", allow-prereleases = true } 28 | pytest = "^5.3" 29 | sphinx = "^2.2" 30 | mkdocs = "^1.0" 31 | sphinx_rtd_theme = "^0.4.3" 32 | recommonmark = "^0.6.0" 33 | fn_deps = "^0.1.0" 34 | 35 | [tool.poetry.extras] 36 | examples = [ 37 | "seaborn", 38 | "statsmodels", 39 | "matplotlib", 40 | "sklearn", 41 | "plotly", 42 | "pandas", 43 | "yfinance" 44 | ] 45 | 46 | [tool.dephell.main] 47 | from = { format = "poetry", path = "pyproject.toml" } 48 | to = { format = "setuppy", path = "setup.py" } 49 | 50 | [build-system] 51 | requires = ["poetry>=0.12"] 52 | build-backend = "poetry.masonry.api" 53 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | 2 | # -*- coding: utf-8 -*- 3 | 4 | # DO NOT EDIT THIS FILE! 5 | # This file has been autogenerated by dephell <3 6 | # https://github.com/dephell/dephell 7 | 8 | try: 9 | from setuptools import setup 10 | except ImportError: 11 | from distutils.core import setup 12 | 13 | 14 | import os.path 15 | 16 | readme = '' 17 | here = os.path.abspath(os.path.dirname(__file__)) 18 | readme_path = os.path.join(here, 'README.rst') 19 | if os.path.exists(readme_path): 20 | with open(readme_path, 'rb') as stream: 21 | readme = stream.read().decode('utf8') 22 | 23 | 24 | setup( 25 | long_description=readme, 26 | name='fn_graph', 27 | version='0.14.3', 28 | description='Manage, maintain and reuse complex function graphs without the hassle.', 29 | python_requires='==3.*,>=3.7.0', 30 | project_urls={"documentation": "https://fn-graph.readthedocs.io/", "homepage": "https://github.com/BusinessOptics/fn_graph", "repository": "https://github.com/BusinessOptics/fn_graph"}, 31 | author='James Saunders', 32 | author_email='james@businessoptics.biz', 33 | license='MIT', 34 | packages=['fn_graph', 'fn_graph.examples', 'fn_graph.tests'], 35 | package_dir={"": "."}, 36 | package_data={"fn_graph.examples": ["*.csv"]}, 37 | install_requires=['graphviz==0.*,>=0.13.2', 'littleutils==0.*,>=0.2.1', 'networkx==2.*,>=2.4.0'], 38 | extras_require={"dev": ["black==18.*,>=18.3.0.a0", "fn-deps==0.*,>=0.1.0", "mkdocs==1.*,>=1.0.0", "pytest==5.*,>=5.3.0", "recommonmark==0.*,>=0.6.0", "sphinx==2.*,>=2.2.0", "sphinx-rtd-theme==0.*,>=0.4.3"], "examples": ["matplotlib", "pandas", "plotly", "seaborn", "sklearn", "statsmodels", "yfinance"]}, 39 | ) 40 | --------------------------------------------------------------------------------