├── .github
    └── workflows
    │   └── ci.yaml
├── .gitignore
├── .pytype
    └── .ninja_deps
├── .vscode
    ├── .ropeproject
    │   └── config.py
    ├── cspell.json
    ├── launch.json
    └── settings.json
├── CHANGES.rst
├── LICENSE
├── MANIFEST.in
├── README.rst
├── bin
    ├── build.sh
    ├── check_branch.sh
    └── check_readme.sh
├── codecov.yml
├── docs
    ├── GraphtikFlowchart.dot
    ├── Makefile
    └── source
    │   ├── GraphtikModuleDependencies.dot
    │   ├── _static
    │       ├── enactSvgPanZoom.js
    │       └── s5defs.css
    │   ├── arch.rst
    │   ├── changes.rst
    │   ├── conf.py
    │   ├── docutils.conf
    │   ├── genindex.rst
    │   ├── images
    │       ├── GraphkitUsageOverview.svg
    │       ├── GraphtikFlowchart-v1.2.4.svg
    │       ├── GraphtikFlowchart-v1.3.0.svg
    │       ├── GraphtikFlowchart-v4.1.0.svg
    │       ├── GraphtikFlowchart-v4.4.0.svg
    │       ├── GraphtikLegend.svg
    │       ├── barebone_3ops.svg
    │       ├── executed_3ops.svg
    │       ├── graphtik-module_deps-v10.4.0.svg
    │       ├── sample.svg
    │       └── unpruned_useless_provides.svg
    │   ├── index.rst
    │   ├── operations.rst
    │   ├── pipelines.rst
    │   ├── plotting.rst
    │   └── reference.rst
├── graphtik
    ├── __init__.py
    ├── autograph.py
    ├── base.py
    ├── config.py
    ├── execution.py
    ├── fnop.py
    ├── jetsam.py
    ├── jsonpointer.py
    ├── modifier.py
    ├── pipeline.py
    ├── planning.py
    ├── plot.py
    ├── py.typed
    └── sphinxext
    │   ├── __init__.py
    │   ├── _graphtikbuilder.py
    │   ├── doctestglobs.py
    │   └── graphtik.css
├── pyproject.toml
├── requirements-rtd.txt
├── requirements.txt
├── setup.cfg
├── setup.py
└── test
    ├── __init__.py
    ├── conftest.py
    ├── helpers.py
    ├── sphinxext
        ├── __init__.py
        ├── conftest.py
        ├── roots
        │   └── test-graphtik-directive
        │   │   ├── conf.py
        │   │   └── index.rst
        ├── test_directive.py
        └── test_image_purgatory.py
    ├── test_base.py
    ├── test_combine.py
    ├── test_execution.py
    ├── test_graphtik.py
    ├── test_hierarchical.py
    ├── test_jsonpointer.py
    ├── test_modifier.py
    ├── test_op.py
    ├── test_planning.py
    ├── test_plot.py
    ├── test_remerge.py
    ├── test_sideffects.py
    └── test_site.py


/.github/workflows/ci.yaml:
--------------------------------------------------------------------------------
 1 | name: test-n-release
 2 | 
 3 | on:
 4 |   push:
 5 |   pull_request:
 6 |   release:
 7 |     types: [published]
 8 | 
 9 | jobs:
10 |   build-n-publish:
11 |     runs-on: ubuntu-latest
12 |     continue-on-error: ${{ matrix.experimental }}
13 |     strategy:
14 |       matrix:
15 |         python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
16 |         experimental: [false]
17 |         include:
18 |           - version: "3.12-dev"
19 |             experimental: true
20 | 
21 |     steps:
22 |       - uses: actions/checkout@v3
23 | 
24 |       - name: Set up Python ${{ matrix.python-version }}
25 |         uses: actions/setup-python@v4
26 |         with:
27 |           python-version: ${{ matrix.python-version }}
28 |           cache: "pip"
29 | 
30 |       - name: graphviz
31 |         uses: ts-graphviz/setup-graphviz@v1
32 | 
33 |       - name: install
34 |         run: |
35 |           python -m pip install --upgrade pip
36 |           pip install -e .[all]
37 | 
38 |       - name: test
39 |         if: matrix.python-version != '3.11'
40 |         run: |
41 |           # Undo configs in setup.cfg
42 |           echo -e '[pytest]\nmarkers: slow' > pytest.ini
43 |           pytest --cov=graphtik #--log-level=DEBUG -v
44 | 
45 |       - name: test-slow
46 |         if: matrix.python-version == '3.11'
47 |         run: |
48 |           pytest -m 'slow or not slow' --cov=graphtik #--log-level=DEBUG -v
49 | 
50 |       - name: upload@codecov
51 |         uses: codecov/codecov-action@v3
52 |         with:
53 |           env_vars: OS,PYTHON
54 |         #   fail_ci_if_error: true # optional (default = false)
55 |         #   verbose: true # optional (default = false)
56 | 
57 |       - name: build
58 |         run: |
59 |           python -m build
60 | 
61 |       - name: publish@test-pypi
62 |         if: >
63 |           github.event_name == 'release' &&
64 |           matrix.python-version == '3.11' &&
65 |           github.repository_owner != 'pygraphkit'
66 |         uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
67 |         with:
68 |           repository_url: https://test.pypi.org/legacy/
69 |           password: ${{ secrets.TEST_PYPI_API_TOKEN }}
70 | 
71 |       - name: publish@pypi
72 |         if: >
73 |           github.event_name == 'release' &&
74 |           matrix.python-version == '3.11' &&
75 |           github.repository_owner == 'pygraphkit'
76 |         uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
77 |         with:
78 |           password: ${{ secrets.PYPI_API_TOKEN }}
79 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # Byte-compiled / optimized / DLL files
 2 | __pycache__/
 3 | *.py[cod]
 4 | 
 5 | # C extensions
 6 | *.so
 7 | 
 8 | # Distribution / packaging
 9 | .Python
10 | /env*/
11 | /.env*/
12 | /venv*/
13 | /.venv*/
14 | build/
15 | develop-eggs/
16 | dist/
17 | downloads/
18 | eggs/
19 | lib/
20 | lib64/
21 | parts/
22 | sdist/
23 | var/
24 | *.egg-info/
25 | .installed.cfg
26 | *.egg
27 | 
28 | # PyInstaller
29 | #  Usually these files are written by a python script from a template
30 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
31 | *.manifest
32 | *.spec
33 | 
34 | # Installer logs
35 | pip-log.txt
36 | pip-delete-this-directory.txt
37 | 
38 | # Unit test / coverage reports
39 | htmlcov/
40 | .tox/
41 | .coverage
42 | .cache
43 | nosetests.xml
44 | coverage.xml
45 | .coverage*
46 | 
47 | # Translations
48 | *.mo
49 | *.pot
50 | 
51 | # Django stuff:
52 | *.log
53 | 
54 | # Sphinx documentation
55 | docs/_build/
56 | 
57 | # PyBuilder
58 | target/
59 | 
60 | # Plots genersated when running sample code
61 | /*.png
62 | /*.svg
63 | /*.pdf


--------------------------------------------------------------------------------
/.pytype/.ninja_deps:
--------------------------------------------------------------------------------
1 | # ninjadeps
2 |    


--------------------------------------------------------------------------------
/.vscode/.ropeproject/config.py:
--------------------------------------------------------------------------------
  1 | # The default ``config.py``
  2 | # flake8: noqa
  3 | 
  4 | 
  5 | def set_prefs(prefs):
  6 |     """This function is called before opening the project"""
  7 | 
  8 |     # Specify which files and folders to ignore in the project.
  9 |     # Changes to ignored resources are not added to the history and
 10 |     # VCSs.  Also they are not returned in `Project.get_files()`.
 11 |     # Note that ``?`` and ``*`` match all characters but slashes.
 12 |     # '*.pyc': matches 'test.pyc' and 'pkg/test.pyc'
 13 |     # 'mod*.pyc': matches 'test/mod1.pyc' but not 'mod/1.pyc'
 14 |     # '.svn': matches 'pkg/.svn' and all of its children
 15 |     # 'build/*.o': matches 'build/lib.o' but not 'build/sub/lib.o'
 16 |     # 'build//*.o': matches 'build/lib.o' and 'build/sub/lib.o'
 17 |     prefs['ignored_resources'] = ['*.pyc', '*~', '.ropeproject',
 18 |                                   '.hg', '.svn', '_svn', '.git', '.tox']
 19 | 
 20 |     # Specifies which files should be considered python files.  It is
 21 |     # useful when you have scripts inside your project.  Only files
 22 |     # ending with ``.py`` are considered to be python files by
 23 |     # default.
 24 |     # prefs['python_files'] = ['*.py']
 25 | 
 26 |     # Custom source folders:  By default rope searches the project
 27 |     # for finding source folders (folders that should be searched
 28 |     # for finding modules).  You can add paths to that list.  Note
 29 |     # that rope guesses project source folders correctly most of the
 30 |     # time; use this if you have any problems.
 31 |     # The folders should be relative to project root and use '/' for
 32 |     # separating folders regardless of the platform rope is running on.
 33 |     # 'src/my_source_folder' for instance.
 34 |     # prefs.add('source_folders', 'src')
 35 | 
 36 |     # You can extend python path for looking up modules
 37 |     # prefs.add('python_path', '~/python/')
 38 | 
 39 |     # Should rope save object information or not.
 40 |     prefs['save_objectdb'] = True
 41 |     prefs['compress_objectdb'] = False
 42 | 
 43 |     # If `True`, rope analyzes each module when it is being saved.
 44 |     prefs['automatic_soa'] = True
 45 |     # The depth of calls to follow in static object analysis
 46 |     prefs['soa_followed_calls'] = 0
 47 | 
 48 |     # If `False` when running modules or unit tests "dynamic object
 49 |     # analysis" is turned off.  This makes them much faster.
 50 |     prefs['perform_doa'] = True
 51 | 
 52 |     # Rope can check the validity of its object DB when running.
 53 |     prefs['validate_objectdb'] = True
 54 | 
 55 |     # How many undos to hold?
 56 |     prefs['max_history_items'] = 32
 57 | 
 58 |     # Shows whether to save history across sessions.
 59 |     prefs['save_history'] = True
 60 |     prefs['compress_history'] = False
 61 | 
 62 |     # Set the number spaces used for indenting.  According to
 63 |     # :PEP:`8`, it is best to use 4 spaces.  Since most of rope's
 64 |     # unit-tests use 4 spaces it is more reliable, too.
 65 |     prefs['indent_size'] = 4
 66 | 
 67 |     # Builtin and c-extension modules that are allowed to be imported
 68 |     # and inspected by rope.
 69 |     prefs['extension_modules'] = []
 70 | 
 71 |     # Add all standard c-extensions to extension_modules list.
 72 |     prefs['import_dynload_stdmods'] = True
 73 | 
 74 |     # If `True` modules with syntax errors are considered to be empty.
 75 |     # The default value is `False`; When `False` syntax errors raise
 76 |     # `rope.base.exceptions.ModuleSyntaxError` exception.
 77 |     prefs['ignore_syntax_errors'] = False
 78 | 
 79 |     # If `True`, rope ignores unresolvable imports.  Otherwise, they
 80 |     # appear in the importing namespace.
 81 |     prefs['ignore_bad_imports'] = False
 82 | 
 83 |     # If `True`, rope will insert new module imports as
 84 |     # `from <package> import <module>` by default.
 85 |     prefs['prefer_module_from_imports'] = False
 86 | 
 87 |     # If `True`, rope will transform a comma list of imports into
 88 |     # multiple separate import statements when organizing
 89 |     # imports.
 90 |     prefs['split_imports'] = False
 91 | 
 92 |     # If `True`, rope will remove all top-level import statements and
 93 |     # reinsert them at the top of the module when making changes.
 94 |     prefs['pull_imports_to_top'] = True
 95 | 
 96 |     # If `True`, rope will sort imports alphabetically by module name instead
 97 |     # of alphabetically by import statement, with from imports after normal
 98 |     # imports.
 99 |     prefs['sort_imports_alphabetically'] = False
100 | 
101 |     # Location of implementation of
102 |     # rope.base.oi.type_hinting.interfaces.ITypeHintingFactory In general
103 |     # case, you don't have to change this value, unless you're an rope expert.
104 |     # Change this value to inject you own implementations of interfaces
105 |     # listed in module rope.base.oi.type_hinting.providers.interfaces
106 |     # For example, you can add you own providers for Django Models, or disable
107 |     # the search type-hinting in a class hierarchy, etc.
108 |     prefs['type_hinting_factory'] = (
109 |         'rope.base.oi.type_hinting.factory.default_type_hinting_factory')
110 | 
111 | 
112 | def project_opened(project):
113 |     """This function is called after opening the project"""
114 |     # Do whatever you like here!
115 | 


--------------------------------------------------------------------------------
/.vscode/cspell.json:
--------------------------------------------------------------------------------
  1 | {
  2 |     "version": "0.1",
  3 |     "language": "en",
  4 |     "enabled": true,
  5 |     "ignoreRegExpList": [
  6 |         "'s\\b"
  7 |     ],
  8 |     "ignorePaths": [
  9 |         ".git/",
 10 |         "*venv*/",
 11 |         "build/",
 12 |         "dist/",
 13 |         "*.egg-info/",
 14 |         ".vscode/",
 15 |         "__*",
 16 |         "*scratch*"
 17 |     ],
 18 |     "words": [
 19 |         "abspow",
 20 |         "acallable",
 21 |         "addinivalue",
 22 |         "addmul",
 23 |         "anagnostopoulos",
 24 |         "ankostis",
 25 |         "arel",
 26 |         "asdict",
 27 |         "astuple",
 28 |         "autodoc",
 29 |         "autodocs",
 30 |         "baumgartner",
 31 |         "Berman",
 32 |         "BGCOLOR",
 33 |         "blackline",
 34 |         "boltons",
 35 |         "bools",
 36 |         "bugfix",
 37 |         "builtins",
 38 |         "calced",
 39 |         "callables",
 40 |         "calllbacks",
 41 |         "changelog",
 42 |         "clsname",
 43 |         "cmap",
 44 |         "cmapx",
 45 |         "codecov",
 46 |         "codestyle",
 47 |         "concat",
 48 |         "conds",
 49 |         "configs",
 50 |         "confval",
 51 |         "conv",
 52 |         "cordero",
 53 |         "covid",
 54 |         "cstor",
 55 |         "Dask",
 56 |         "dataframe",
 57 |         "dataframes",
 58 |         "datanodes",
 59 |         "dcore",
 60 |         "dedented",
 61 |         "dedupe",
 62 |         "deduping",
 63 |         "deps",
 64 |         "dilled",
 65 |         "docstring",
 66 |         "doctest",
 67 |         "doctesting",
 68 |         "doctests",
 69 |         "dogfood",
 70 |         "dogfooded",
 71 |         "dogfoods",
 72 |         "dtype",
 73 |         "dtypes",
 74 |         "dynaimage",
 75 |         "envvar",
 76 |         "errgraph",
 77 |         "errored",
 78 |         "etall",
 79 |         "evalcontextfilter",
 80 |         "exemethod",
 81 |         "exinfo",
 82 |         "favicon",
 83 |         "figclass",
 84 |         "figwidth",
 85 |         "fillcolor",
 86 |         "fillna",
 87 |         "filtdict",
 88 |         "fname",
 89 |         "fnop",
 90 |         "fontname",
 91 |         "fpath",
 92 |         "fpaths",
 93 |         "fqdn",
 94 |         "funcs",
 95 |         "functools",
 96 |         "garrigues",
 97 |         "genindex",
 98 |         "getenv",
 99 |         "graphkit",
100 |         "graphop",
101 |         "graphtik",
102 |         "graphvar",
103 |         "graphviz",
104 |         "Hashable",
105 |         "hcat",
106 |         "hnguyen",
107 |         "howto",
108 |         "hpgl",
109 |         "hrefer",
110 |         "htaccess",
111 |         "Huygn",
112 |         "huyng",
113 |         "imap",
114 |         "implicits",
115 |         "imread",
116 |         "imshow",
117 |         "infos",
118 |         "inodes",
119 |         "inps",
120 |         "inversed",
121 |         "invis",
122 |         "isbuiltin",
123 |         "iset",
124 |         "ismap",
125 |         "issubset",
126 |         "iterables",
127 |         "itertools",
128 |         "ized",
129 |         "jetsamed",
130 |         "jsonpized",
131 |         "jsonpointer",
132 |         "jupy",
133 |         "jupyter",
134 |         "kwargs",
135 |         "maplotlib",
136 |         "marshalled",
137 |         "marshalling",
138 |         "matplolib",
139 |         "matplot",
140 |         "melticulously",
141 |         "mergeable",
142 |         "misattributed",
143 |         "monkeypatch",
144 |         "monkeypatching",
145 |         "mpimg",
146 |         "msgs",
147 |         "multithread",
148 |         "multithreaded",
149 |         "multithreading",
150 |         "myadd",
151 |         "mygraph",
152 |         "mykws",
153 |         "mypkg",
154 |         "mypow",
155 |         "nbunch",
156 |         "ndarray",
157 |         "ndiff",
158 |         "netop",
159 |         "netops",
160 |         "networkx",
161 |         "nexpected",
162 |         "nfewer",
163 |         "ngot",
164 |         "nmiss",
165 |         "nsharks",
166 |         "numpy",
167 |         "obox",
168 |         "odiamond",
169 |         "odot",
170 |         "opargs",
171 |         "opbuilder",
172 |         "opname",
173 |         "opneeds",
174 |         "opprovides",
175 |         "opsattrs",
176 |         "optionalize",
177 |         "ortho",
178 |         "outp",
179 |         "overridable",
180 |         "overspill",
181 |         "overspilled",
182 |         "overwriting",
183 |         "parallelizable",
184 |         "pdfs",
185 |         "pdot",
186 |         "pext",
187 |         "pformat",
188 |         "picklable",
189 |         "plotly",
190 |         "plottable",
191 |         "plottables",
192 |         "pngs",
193 |         "provenanced",
194 |         "pumpikano",
195 |         "pydot",
196 |         "pygraphkit",
197 |         "pylint",
198 |         "pyplot",
199 |         "pytest",
200 |         "pytestmark",
201 |         "pyversion",
202 |         "qmark",
203 |         "qname",
204 |         "qubed",
205 |         "questionmark",
206 |         "quickstart",
207 |         "rebased",
208 |         "rebranded",
209 |         "recursed",
210 |         "refact",
211 |         "regexes",
212 |         "remerge",
213 |         "remerged",
214 |         "reparse",
215 |         "repr",
216 |         "resched",
217 |         "rescheduler",
218 |         "restruct",
219 |         "retarget",
220 |         "retargeting",
221 |         "robwhess",
222 |         "schedula",
223 |         "screamy",
224 |         "sdist",
225 |         "seealso",
226 |         "sfxed",
227 |         "sideffect",
228 |         "sideffected",
229 |         "sideffecteds",
230 |         "sideffects",
231 |         "sidefx",
232 |         "signularize",
233 |         "signulars",
234 |         "skipif",
235 |         "solaris",
236 |         "sphinxcontrib",
237 |         "sphinxext",
238 |         "splitted",
239 |         "stackable",
240 |         "stdout",
241 |         "subdoc",
242 |         "subdocs",
243 |         "subgraph",
244 |         "subgraphs",
245 |         "substs",
246 |         "sucessors",
247 |         "superdoc",
248 |         "superdocs",
249 |         "svgs",
250 |         "svgz",
251 |         "syamajala",
252 |         "theming",
253 |         "tobi",
254 |         "tocs",
255 |         "todel",
256 |         "todos",
257 |         "tooltips",
258 |         "topo",
259 |         "Traceback",
260 |         "travis",
261 |         "tristates",
262 |         "truthies",
263 |         "tuplized",
264 |         "unsubscriptable",
265 |         "upnext",
266 |         "upstreams",
267 |         "usefull",
268 |         "utils",
269 |         "vals",
270 |         "vararg",
271 |         "varargish",
272 |         "varargs",
273 |         "varname",
274 |         "vcat",
275 |         "withset",
276 |         "writtable",
277 |         "xfail",
278 |         "xfailif",
279 |         "xlib",
280 |         "zoomable"
281 |     ],
282 |     "flagWords": [
283 |         "errorred"
284 |     ]
285 | }


--------------------------------------------------------------------------------
/.vscode/launch.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     // Use IntelliSense to learn about possible attributes.
 3 |     // Hover to view descriptions of existing attributes.
 4 |     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
 5 |     "version": "0.2.0",
 6 |     "configurations": [
 7 |         {
 8 |             "name": "Python: Current File",
 9 |             "type": "python",
10 |             "request": "launch",
11 |             "program": "${file}",
12 |             "console": "integratedTerminal"
13 |         },
14 |         {
15 |             "name": "Python: TEST",
16 |             "type": "python",
17 |             "request": "test",
18 |             "justMyCode": false,
19 |             "redirectOutput": false,
20 |             "env": {
21 |                 "PYTEST_ADDOPTS": "-vv -s --log-level=DEBUG"
22 |             },
23 |             "console": "integratedTerminal"
24 |         }
25 |     ]
26 | }


--------------------------------------------------------------------------------
/.vscode/settings.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "python.pythonPath": ".venv/bin/python",
 3 |     "python.formatting.provider": "black",
 4 |     "files.trimFinalNewlines": true,
 5 |     "files.trimTrailingWhitespace": true,
 6 |     "editor.formatOnSave": true,
 7 |     "editor.codeActionsOnSave": {
 8 |         "source.organizeImports": true,
 9 |     },
10 |     "python.testing.unittestEnabled": false,
11 |     "python.testing.nosetestsEnabled": false,
12 |     "python.testing.pytestEnabled": true,
13 |     "editor.formatOnSaveTimeout": 1500,
14 |     "files.watcherExclude": {
15 |         "**/.git/objects/**": true,
16 |         "**/.git/subtree-cache/**": true,
17 |         "/*venv*/**": true,
18 |         "**/*cache*/**": true,
19 |         "**/.pytype/**": true,
20 |         "**/.ipynb_checkpoints/": true,
21 |         "**/build/**": true,
22 |         "**/.coverage*": true
23 |     },
24 |     "restructuredtext.confPath": ""
25 | }


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 | Apache License
  2 | Version 2.0, January 2004
  3 | 
  4 | http://www.apache.org/licenses/
  5 | 
  6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  7 | 
  8 | 1. Definitions.
  9 | "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by
 10 | Sections 1 through 9 of this document.
 11 | 
 12 | "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting
 13 | the License.
 14 | 
 15 | "Legal Entity" shall mean the union of the acting entity and all other entities that control, are
 16 | controlled by, or are under common control with that entity. For the purposes of this definition,
 17 | "control" means (i) the power, direct or indirect, to cause the direction or management of such entity,
 18 | whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding
 19 | shares, or (iii) beneficial ownership of such entity.
 20 | 
 21 | "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License.
 22 | 
 23 | "Source" form shall mean the preferred form for making modifications, including but not limited to software
 24 | source code, documentation source, and configuration files.
 25 | 
 26 | "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form,
 27 | including but not limited to compiled object code, generated documentation, and conversions to other media types.
 28 | 
 29 | "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License,
 30 | as indicated by a copyright notice that is included in or attached to the work (an example is provided in
 31 | the Appendix below).
 32 | 
 33 | "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from)
 34 | the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent,
 35 | as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not
 36 | include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work
 37 | and Derivative Works thereof.
 38 | 
 39 | "Contribution" shall mean any work of authorship, including the original version of the Work and any
 40 | modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to
 41 | Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to
 42 | submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of
 43 | electronic, verbal, or written communication sent to the Licensor or its representatives, including but not
 44 | limited to communication on electronic mailing lists, source code control systems, and issue tracking systems
 45 | that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but
 46 | excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner
 47 | as "Not a Contribution."
 48 | 
 49 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been
 50 | received by Licensor and subsequently incorporated within the Work.
 51 | 
 52 | 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby
 53 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license
 54 | to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute
 55 | the Work and such Derivative Works in Source or Object form.
 56 | 
 57 | 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby
 58 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated
 59 | in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer
 60 | the Work, where such license applies only to those patent claims licensable by such Contributor that are
 61 | necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work
 62 | to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including
 63 | a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the
 64 | Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under
 65 | this License for that Work shall terminate as of the date such litigation is filed.
 66 | 
 67 | 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any
 68 | medium, with or without modifications, and in Source or Object form, provided that You meet the following
 69 | conditions:
 70 | 
 71 | You must give any other recipients of the Work or Derivative Works a copy of this License; and
 72 | You must cause any modified files to carry prominent notices stating that You changed the files; and
 73 | You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent,
 74 | trademark, and attribution notices from the Source form of the Work, excluding those notices that do not
 75 | pertain to any part of the Derivative Works; and
 76 | If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You
 77 | distribute must include a readable copy of the attribution notices contained within such NOTICE file,
 78 | excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the
 79 | following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source
 80 | form or documentation, if provided along with the Derivative Works; or, within a display generated by the
 81 | Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file
 82 | are for informational purposes only and do not modify the License. You may add Your own attribution notices
 83 | within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work,
 84 | provided that such additional attribution notices cannot be construed as modifying the License.
 85 | 
 86 | You may add Your own copyright statement to Your modifications and may provide additional or different
 87 | license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such
 88 | Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise
 89 | complies with the conditions stated in this License.
 90 | 
 91 | 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally
 92 | submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this
 93 | License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall
 94 | supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding
 95 | such Contributions.
 96 | 
 97 | 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or
 98 | product names of the Licensor, except as required for reasonable and customary use in describing the origin
 99 | of the Work and reproducing the content of the NOTICE file.
100 | 
101 | 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the
102 | Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
103 | OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE,
104 | NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for
105 | determining the appropriateness of using or redistributing the Work and assume any risks associated with Your
106 | exercise of permissions under this License.
107 | 
108 | 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence),
109 | contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts)
110 | or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect,
111 | special, incidental, or consequential damages of any character arising as a result of this License or out
112 | of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work
113 | stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such
114 | Contributor has been advised of the possibility of such damages.
115 | 
116 | 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof,
117 | You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other
118 | liability obligations and/or rights consistent with this License. However, in accepting such obligations,
119 | You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor,
120 | and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred
121 | by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional
122 | liability.
123 | 
124 | END OF TERMS AND CONDITIONS
125 | 


--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | include LICENSE CHANGES.rst
2 | 


--------------------------------------------------------------------------------
/bin/build.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | # clean, or invalid files in packages
4 | rm -vrf ./build/* ./dist/* ./*.pyc ./*.tgz ./*.egg-info
5 | python -m build
6 | sphinx-build -Wj auto -D graphtik_warning_is_error=true docs/source/ docs/build/


--------------------------------------------------------------------------------
/bin/check_branch.sh:
--------------------------------------------------------------------------------
 1 | #! /bin/bash
 2 | #
 3 | # SYNTAX: check-branch.sh [branch]
 4 | #
 5 | # Run it in a cloned repo to clean-test recent master
 6 | 
 7 | my_dir=`dirname "$0"`
 8 | cd "$my_dir/.."
 9 | rm -rf build/* && git fetch origin && git reset --hard origin/${1:-master} && pytest
10 | 


--------------------------------------------------------------------------------
/bin/check_readme.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | #-*- coding: utf-8 -*-
 3 | #
 4 | # Copyright 2013-2019 European Commission (JRC);
 5 | # Licensed under the EUPL (the 'Licence');
 6 | # You may not use this work except in compliance with the Licence.
 7 | # You may obtain a copy of the Licence at: http://ec.europa.eu/idabc/eupl
 8 | 
 9 | 
10 | ## Checks that README has no RsT-syntactic errors.
11 | # Since it is used by `setup.py`'s `description` if it has any errors, 
12 | # PyPi would fail parsing them, ending up with an ugly landing page,
13 | # when uploaded.
14 | 
15 | set +x  # Set -x for debugging script.
16 | 
17 | my_dir=`dirname "$0"`
18 | my_name=`basename "$0"`
19 | my_dir="$(dirname "$0")"
20 | if [ -f "$my_dir/../setup.py" ]; then
21 |         cd "$my_dir/.."
22 | else
23 |         my_dir="$PWD"
24 | fi
25 | 
26 | tmp_dir=$(mktemp -d -t wltp-$my_name-XXXXXXXXXX)
27 | 
28 | rm -rf "build/*" "dist/*"
29 | python setup.py -q bdist_wheel --dist-dir $tmp_dir && \
30 | twine check $tmp_dir/*.whl 
31 | 
32 | exc=$?
33 | rm -rf $tmp_dir
34 | exit $exc
35 | 


--------------------------------------------------------------------------------
/codecov.yml:
--------------------------------------------------------------------------------
1 | # It just needs to exist for codecov to annotate PRs.
2 | 


--------------------------------------------------------------------------------
/docs/GraphtikFlowchart.dot:
--------------------------------------------------------------------------------
 1 | # Render it manually with this command, and remember to update result in git:
 2 | #
 3 | #   dot docs/GraphtikFlowchart.dot -Tsvg -odocs/source/images/GraphtikFlowchart-vX.Y.Z.svg
 4 | #
 5 | digraph {
 6 |     label="graphtik-v4.4+ flowchart";
 7 |     labelloc=t;
 8 | 
 9 |     operations  [shape=parallelogram fontname="italic" tooltip=class
10 |                  URL="arch.html#term-operation"];
11 |     compose     [fontname="italic" tooltip=phase
12 |                  URL="arch.html#term-composition"];
13 |     network     [shape=parallelogram fontname="italic" tooltip=class
14 |                  URL="arch.html#term-network"];
15 |     inputs      [shape=rect label="input names" tooltip=mappings
16 |                  URL="arch.html#term-inputs"];
17 |     outputs     [shape=rect label="output names" tooltip=mappings
18 |                  URL="arch.html#term-outputs"];
19 |     predicate   [shape=rect label="node predicate" tooltip=function
20 |                  URL="arch.html#term-node-predicate"];
21 |     subgraph cluster_compute {
22 |         label=compute
23 |         fontname=bold
24 |         style=dashed
25 |         labelloc=b
26 |         tooltip="process, NetOp's method"
27 |         URL="arch.html#term-compute"
28 | 
29 |         compile     [fontname="italic" tooltip="phase, Network's method"
30 |                      URL="arch.html#term-compilation"];
31 |         plan        [shape=parallelogram label="execution plan" fontname="italic" tooltip=class
32 |                      URL="arch.html#term-execution-plan"];
33 |         execute     [fontname=italic fontname="italic" tooltip="phase, Plan's method"
34 |                      URL="arch.html#term-execution"];
35 |     }
36 |     values      [shape=rect label="input values" tooltip=mappings
37 |                  URL="arch.html#term-inputs"];
38 |     solution    [shape=parallelogram tooltip=class
39 |                  URL="arch.html#term-solution"];
40 | 
41 |     operations -> compose -> network    [arrowhead=vee];
42 |     network -> compile [arrowhead=vee
43 |                         label=<&#9679;graph>
44 |                         tooltip="operations linked by the dependencies"
45 |                         URL="arch.html#term-graph"];
46 |     {inputs outputs predicate} -> compile [arrowhead=vee];
47 |     compile -> plan    [arrowhead=vee
48 |                         label=<&#9679;pruned dag>
49 |                         tooltip="graph pruned by inputs & outputs"
50 |                         URL="arch.html#term-prune"];
51 |     plan -> execute    [arrowhead=vee
52 |                         label=<&#9679;dag>
53 |                         tooltip="plan's dag"
54 |                         URL="arch.html#term-dag"];
55 |     values -> execute [arrowhead=vee];
56 |     execute -> solution    [arrowhead=vee
57 |                             tooltip="dag clone to modify while executing"
58 |                             label=<&#9679;solution dag> URL="arch.html#term-solution-dag"];
59 |     solution -> solution   [arrowhead=vee
60 |                             label=<&#9679;prune dag on reschedule>
61 |                             tooltip="prune when endured operations fail or partial outputs"
62 |                             URL="arch.html#term-reschedule"];
63 | }


--------------------------------------------------------------------------------
/docs/Makefile:
--------------------------------------------------------------------------------
  1 | # Copyright 2016, Yahoo Inc.
  2 | # Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms.
  3 | 
  4 | # Makefile for Sphinx documentation
  5 | #
  6 | 
  7 | # You can set these variables from the command line.
  8 | SPHINXOPTS    =
  9 | SPHINXBUILD   = sphinx-build
 10 | PAPER         =
 11 | BUILDDIR      = build
 12 | 
 13 | # User-friendly check for sphinx-build
 14 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
 15 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
 16 | endif
 17 | 
 18 | # Internal variables.
 19 | PAPEROPT_a4     = -D latex_paper_size=a4
 20 | PAPEROPT_letter = -D latex_paper_size=letter
 21 | ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
 22 | # the i18n builder cannot share the environment and doctrees with the others
 23 | I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
 24 | 
 25 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
 26 | 
 27 | help:
 28 | 	@echo "Please use \`make <target>' where <target> is one of"
 29 | 	@echo "  html       to make standalone HTML files"
 30 | 	@echo "  dirhtml    to make HTML files named index.html in directories"
 31 | 	@echo "  singlehtml to make a single large HTML file"
 32 | 	@echo "  pickle     to make pickle files"
 33 | 	@echo "  json       to make JSON files"
 34 | 	@echo "  htmlhelp   to make HTML files and a HTML help project"
 35 | 	@echo "  qthelp     to make HTML files and a qthelp project"
 36 | 	@echo "  devhelp    to make HTML files and a Devhelp project"
 37 | 	@echo "  epub       to make an epub"
 38 | 	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
 39 | 	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
 40 | 	@echo "  latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
 41 | 	@echo "  text       to make text files"
 42 | 	@echo "  man        to make manual pages"
 43 | 	@echo "  texinfo    to make Texinfo files"
 44 | 	@echo "  info       to make Texinfo files and run them through makeinfo"
 45 | 	@echo "  gettext    to make PO message catalogs"
 46 | 	@echo "  changes    to make an overview of all changed/added/deprecated items"
 47 | 	@echo "  xml        to make Docutils-native XML files"
 48 | 	@echo "  pseudoxml  to make pseudoxml-XML files for display purposes"
 49 | 	@echo "  linkcheck  to check all external links for integrity"
 50 | 	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
 51 | 
 52 | clean:
 53 | 	rm -rf $(BUILDDIR)/*
 54 | 
 55 | html:
 56 | 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 57 | 	@echo
 58 | 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 59 | 
 60 | dirhtml:
 61 | 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
 62 | 	@echo
 63 | 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
 64 | 
 65 | singlehtml:
 66 | 	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
 67 | 	@echo
 68 | 	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
 69 | 
 70 | pickle:
 71 | 	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
 72 | 	@echo
 73 | 	@echo "Build finished; now you can process the pickle files."
 74 | 
 75 | json:
 76 | 	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
 77 | 	@echo
 78 | 	@echo "Build finished; now you can process the JSON files."
 79 | 
 80 | htmlhelp:
 81 | 	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
 82 | 	@echo
 83 | 	@echo "Build finished; now you can run HTML Help Workshop with the" \
 84 | 	      ".hhp project file in $(BUILDDIR)/htmlhelp."
 85 | 
 86 | qthelp:
 87 | 	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
 88 | 	@echo
 89 | 	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
 90 | 	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
 91 | 	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/graphkit.qhcp"
 92 | 	@echo "To view the help file:"
 93 | 	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/graphkit.qhc"
 94 | 
 95 | devhelp:
 96 | 	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
 97 | 	@echo
 98 | 	@echo "Build finished."
 99 | 	@echo "To view the help file:"
100 | 	@echo "# mkdir -p $$HOME/.local/share/devhelp/graphkit"
101 | 	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/graphkit"
102 | 	@echo "# devhelp"
103 | 
104 | epub:
105 | 	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
106 | 	@echo
107 | 	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
108 | 
109 | latex:
110 | 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
111 | 	@echo
112 | 	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
113 | 	@echo "Run \`make' in that directory to run these through (pdf)latex" \
114 | 	      "(use \`make latexpdf' here to do that automatically)."
115 | 
116 | latexpdf:
117 | 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
118 | 	@echo "Running LaTeX files through pdflatex..."
119 | 	$(MAKE) -C $(BUILDDIR)/latex all-pdf
120 | 	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
121 | 
122 | latexpdfja:
123 | 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
124 | 	@echo "Running LaTeX files through platex and dvipdfmx..."
125 | 	$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
126 | 	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
127 | 
128 | text:
129 | 	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
130 | 	@echo
131 | 	@echo "Build finished. The text files are in $(BUILDDIR)/text."
132 | 
133 | man:
134 | 	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
135 | 	@echo
136 | 	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
137 | 
138 | texinfo:
139 | 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
140 | 	@echo
141 | 	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
142 | 	@echo "Run \`make' in that directory to run these through makeinfo" \
143 | 	      "(use \`make info' here to do that automatically)."
144 | 
145 | info:
146 | 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
147 | 	@echo "Running Texinfo files through makeinfo..."
148 | 	make -C $(BUILDDIR)/texinfo info
149 | 	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
150 | 
151 | gettext:
152 | 	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
153 | 	@echo
154 | 	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
155 | 
156 | changes:
157 | 	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
158 | 	@echo
159 | 	@echo "The overview file is in $(BUILDDIR)/changes."
160 | 
161 | linkcheck:
162 | 	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
163 | 	@echo
164 | 	@echo "Link check complete; look for any errors in the above output " \
165 | 	      "or in $(BUILDDIR)/linkcheck/output.txt."
166 | 
167 | doctest:
168 | 	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
169 | 	@echo "Testing of doctests in the sources finished, look at the " \
170 | 	      "results in $(BUILDDIR)/doctest/output.txt."
171 | 
172 | xml:
173 | 	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
174 | 	@echo
175 | 	@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
176 | 
177 | pseudoxml:
178 | 	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
179 | 	@echo
180 | 	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
181 | 


--------------------------------------------------------------------------------
/docs/source/GraphtikModuleDependencies.dot:
--------------------------------------------------------------------------------
 1 | digraph {
 2 |     label="graphtik-v8.3.1+ module dependencies";
 3 |     labelloc=t;
 4 |     tooltip="graphtik module dependencies";
 5 |     nodesep=0.55;
 6 |     remincross=true;
 7 |     node [target="_top" style=filled];
 8 |     edge [target="_top"];
 9 |     graph [rankdir=TB URL="../reference.html", target=_top];
10 | 
11 |     subgraph cluster_user_API {
12 |         label="user API";
13 |         labelloc=t;
14 |         rank=S;
15 |         tooltip="modules for clients to interact with";
16 |         "plot.py"        [shape=component
17 |                     tooltip="(extra) plot graphs"
18 |                     fillcolor=Aquamarine
19 |                     URL="../reference.html#module-graphtik.plot"];
20 |         "sphinxext/"   [shape=tab
21 |                     tooltip="(extra) package & modules for plotting graph in Sphinx sites"
22 |                     fillcolor=Aquamarine
23 |                     URL="../reference.html#module-graphtik.sphinxext"];
24 | 
25 |         "pipeline.py"    [shape=component
26 |                     tooltip="(public)"
27 |                     fillcolor=wheat
28 |                     URL="../reference.html#module-graphtik.pipeline"];
29 |         "fnop.py"    [shape=component
30 |                     tooltip="(public)"
31 |                     fillcolor=wheat
32 |                     URL="../reference.html#module-graphtik.fnop"];
33 |         "modifiers.py"   [shape=component
34 |                     tooltip="(public) almost everything imports this module (not-shown)"
35 |                     fillcolor=wheat penwidth=3
36 |                     URL="../reference.html#module-graphtik.modifiers"];
37 | 
38 | 
39 |         subgraph cluster_planning {
40 |             label="core modules";
41 |             labelloc=b;
42 |             tooltip="modules related to graph solution, client is not expected to interact much with them";
43 |             URL="arch.html#term-execution";
44 |             rank=S;
45 | 
46 |             "planning.py"     [shape=component
47 |                         tooltip="(private)"
48 |                         fillcolor=AliceBlue
49 |                         URL="../reference.html#module-graphtik.planning"];
50 |             "execution.py"   [shape=component
51 |                         tooltip="(private)"
52 |                         fillcolor=AliceBlue
53 |                         URL="../reference.html#module-graphtik.execution"];
54 |         }
55 | 
56 |         subgraph cluster_base {
57 |             label="base";
58 |             labelloc=t;
59 |             tooltip="modules imported almost by everything (not shown)";
60 |             rank=min;
61 | 
62 |             "base.py"   [shape=component
63 |                         tooltip="(implicit) everything imports this module (not shown)"
64 |                         fillcolor=wheat penwidth=4
65 |                         URL="../reference.html#module-graphtik.base"];
66 |             "config.py" [shape=component tooltip="(public) almost everything import this module (not shown)"
67 |                         fillcolor=wheat penwidth=4
68 |                         URL="../reference.html#module-graphtik.config"];
69 |         }
70 | 
71 |     }
72 |     subgraph cluster_utils {
73 |         label="utils";
74 |         labelloc=b;
75 |         #rank=S;
76 |         tooltip="almost all other modules depend on these";
77 | 
78 |         "jetsam.py" [shape=component tooltip="utility to annotate exceptions in case of failures"
79 |                     fillcolor=white
80 |                     URL="../reference.html#module-graphtik.jetsam"];
81 |         "jsonpointer.py"      [shape=component tooltip="json-pointer parsing utils"
82 |                     fillcolor=white
83 |                     URL="../reference.html#module-graphtik.jsonpointer"];
84 |     }
85 | 
86 |     #{"fnop.py", "pipeline.py", "planning.py", "execution.py", "plot.py"} -> "base.py"
87 |     #                                        [tooltip="(import-time)"];
88 |     "base.py" -> "plot.py"                  [tooltip="(run-time)" style=dashed];
89 |     {"fnop.py", "pipeline.py", "planning.py", "execution.py"} -> "jetsam.py" [tooltip="(run-time)" style=dashed];
90 |     "execution.py" -> "planning.py"         [tooltip="(import-time)"];
91 |     "planning.py" -> "execution.py"         [tooltip="(run-time)"  style=dashed];
92 |     "pipeline.py" -> "planning.py"          [tooltip="(run-time)" style=dashed];
93 |     "fnop.py" -> "pipeline.py"              [style=dashed tooltip="(run-time) just for plotting"];
94 |     {"modifiers.py", "execution.py", "fnop.py"} -> "jsonpointer.py"           [style=dashed tooltip="(run-time)"];
95 |     "sphinxext/" -> "plot.py"               [tooltip="(import-time)"];
96 | }


--------------------------------------------------------------------------------
/docs/source/_static/enactSvgPanZoom.js:
--------------------------------------------------------------------------------
 1 | function graphtik_armPanZoom() {
 2 |     for (const obj_el of document.querySelectorAll(".graphtik-zoomable-svg")) {
 3 |         svg_el = obj_el.contentDocument.querySelector("svg")
 4 |         var zoom_opts = "graphtikSvgZoomOptions" in obj_el.dataset ?
 5 |             JSON.parse(obj_el.dataset.graphtikSvgZoomOptions) : {};
 6 |         svgPanZoom(svg_el, zoom_opts);
 7 |     };
 8 | };
 9 | 
10 | window.addEventListener("load", graphtik_armPanZoom);
11 | 


--------------------------------------------------------------------------------
/docs/source/_static/s5defs.css:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * CSS hacks and small modification for my Sphinx website
  3 |  * :copyright: Copyright 2013-2016 Lilian Besson
  4 |  * :license: GPLv3, see LICENSE for details.
  5 |  *
  6 |  * Adapted from: https://stackoverflow.com/a/61389938/548792
  7 |  */
  8 | 
  9 | 
 10 | /******
 11 |  * Color roles, for example, :red:`text in RED`.
 12 |  */
 13 | 
 14 | .black {
 15 |     color: black;
 16 | }
 17 | 
 18 | .gray {
 19 |     color: gray;
 20 | }
 21 | 
 22 | .silver {
 23 |     color: silver;
 24 | }
 25 | 
 26 | .white {
 27 |     color: white;
 28 | }
 29 | 
 30 | .maroon {
 31 |     color: maroon;
 32 | }
 33 | 
 34 | .red {
 35 |     color: red;
 36 | }
 37 | 
 38 | .magenta {
 39 |     color: magenta;
 40 | }
 41 | 
 42 | .fuchsia {
 43 |     color: fuchsia;
 44 | }
 45 | 
 46 | .pink {
 47 |     color: pink;
 48 | }
 49 | 
 50 | .orange {
 51 |     color: orange;
 52 | }
 53 | 
 54 | .yellow {
 55 |     color: yellow;
 56 | }
 57 | 
 58 | .lime {
 59 |     color: lime;
 60 | }
 61 | 
 62 | .green {
 63 |     color: green;
 64 | }
 65 | 
 66 | .olive {
 67 |     color: olive;
 68 | }
 69 | 
 70 | .teal {
 71 |     color: teal;
 72 | }
 73 | 
 74 | .cyan {
 75 |     color: cyan;
 76 | }
 77 | 
 78 | .aqua {
 79 |     color: aqua;
 80 | }
 81 | 
 82 | .blue {
 83 |     color: blue;
 84 | }
 85 | 
 86 | .navy {
 87 |     color: navy;
 88 | }
 89 | 
 90 | .purple {
 91 |     color: purple;
 92 | }
 93 | 
 94 | /******
 95 |  * Font-size roles, for example, :huge:`text in big letters`.
 96 |  */
 97 | 
 98 | .tiny {
 99 |     font-size: x-small;
100 | }
101 | 
102 | .small {
103 |     font-size: small;
104 | }
105 | 
106 | .large {
107 |     font-size: large;
108 | }
109 | 
110 | .huge {
111 |     font-size: x-large;
112 | }
113 | .boldit {
114 |     font-weight: bold;
115 |     font-style: italic;
116 | }
117 | 


--------------------------------------------------------------------------------
/docs/source/changes.rst:
--------------------------------------------------------------------------------
 1 | .. include:: ../../CHANGES.rst
 2 |     :end-before: .. _substitutions:
 3 | 
 4 | .. |sample-plot| image:: images/sample.svg
 5 |     :alt: sample graphtik plot
 6 |     :width: 120px
 7 |     :align: bottom
 8 | .. |v1040-module_deps| image:: images/graphtik-module_deps-v10.4.0.svg
 9 |     :alt: graphtik-v10.4.0 module dependencies
10 |     :height: 300px
11 | .. |v410-flowchart| raw:: html
12 |     :file: images/GraphtikFlowchart-v4.1.0.svg
13 | .. |v130-flowchart| image:: images/GraphtikFlowchart-v1.3.0.svg
14 |     :alt: graphtik-v1.3.0 flowchart
15 |     :scale: 75%
16 | .. |v124-flowchart| image:: images/GraphtikFlowchart-v1.2.4.svg
17 |     :alt: graphtik-v1.2.4 flowchart
18 |     :scale: 75%
19 | 


--------------------------------------------------------------------------------
/docs/source/docutils.conf:
--------------------------------------------------------------------------------
1 | # docutils 0.17+ introduce a line-limit by default 10k lines that ius too low for SVG graphs
2 | # eg:
3 | #   docs/source/index.rst:: WARNING: Substitution definition "sample-plot" exceeds the line-length-limit.
4 | # see also https://github.com/spatialaudio/nbsphinx/issues/549
5 | # and here for workaround: https://sourceforge.net/p/docutils/bugs/416/#4d3b
6 | [general]
7 | line_length_limit:100_000_000
8 | 


--------------------------------------------------------------------------------
/docs/source/genindex.rst:
--------------------------------------------------------------------------------
1 | =====
2 | Index
3 | =====
4 | 


--------------------------------------------------------------------------------
/docs/source/images/GraphtikFlowchart-v1.2.4.svg:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 2 | <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
 3 |  "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
 4 | <!-- Generated by graphviz version 2.43.0 (0)
 5 |  -->
 6 | <!-- Title: %3 Pages: 1 -->
 7 | <svg width="309pt" height="355pt"
 8 |  viewBox="0.00 0.00 308.69 355.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
 9 | <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 351)">
10 | <title>%3</title>
11 | <polygon fill="white" stroke="transparent" points="-4,4 -4,-351 304.69,-351 304.69,4 -4,4"/>
12 | <text text-anchor="middle" x="150.35" y="-331.8" font-family="Times,serif" font-size="14.00">graphkit&#45;v1.2.4 flowchart</text>
13 | <!-- operations -->
14 | <g id="node1" class="node">
15 | <title>operations</title>
16 | <polygon fill="none" stroke="black" points="190.53,-324 50.8,-324 14.85,-288 154.59,-288 190.53,-324"/>
17 | <text text-anchor="middle" x="102.69" y="-302.3" font-family="italic" font-size="14.00">operations</text>
18 | </g>
19 | <!-- compose -->
20 | <g id="node2" class="node">
21 | <title>compose</title>
22 | <ellipse fill="none" stroke="black" cx="102.69" cy="-234" rx="102.88" ry="18"/>
23 | <text text-anchor="middle" x="102.69" y="-230.3" font-family="italic" font-size="14.00">compose &amp; compile</text>
24 | </g>
25 | <!-- operations&#45;&gt;compose -->
26 | <g id="edge1" class="edge">
27 | <title>operations&#45;&gt;compose</title>
28 | <path fill="none" stroke="black" d="M102.69,-287.7C102.69,-279.98 102.69,-270.71 102.69,-262.11"/>
29 | <polygon fill="black" stroke="black" points="102.69,-252.1 107.19,-262.1 102.69,-257.1 102.69,-262.1 102.69,-262.1 102.69,-262.1 102.69,-257.1 98.19,-262.1 102.69,-252.1 102.69,-252.1"/>
30 | </g>
31 | <!-- network -->
32 | <g id="node3" class="node">
33 | <title>network</title>
34 | <polygon fill="none" stroke="black" points="175.04,-180 59.95,-180 30.34,-144 145.43,-144 175.04,-180"/>
35 | <text text-anchor="middle" x="102.69" y="-158.3" font-family="italic" font-size="14.00">network</text>
36 | </g>
37 | <!-- compose&#45;&gt;network -->
38 | <g id="edge2" class="edge">
39 | <title>compose&#45;&gt;network</title>
40 | <path fill="none" stroke="black" d="M102.69,-215.7C102.69,-207.98 102.69,-198.71 102.69,-190.11"/>
41 | <polygon fill="black" stroke="black" points="102.69,-180.1 107.19,-190.1 102.69,-185.1 102.69,-190.1 102.69,-190.1 102.69,-190.1 102.69,-185.1 98.19,-190.1 102.69,-180.1 102.69,-180.1"/>
42 | </g>
43 | <!-- compute -->
44 | <g id="node5" class="node">
45 | <title>compute</title>
46 | <ellipse fill="none" stroke="black" cx="174.69" cy="-90" rx="51.19" ry="18"/>
47 | <text text-anchor="middle" x="174.69" y="-86.3" font-family="italic" font-size="14.00">compute</text>
48 | </g>
49 | <!-- network&#45;&gt;compute -->
50 | <g id="edge3" class="edge">
51 | <title>network&#45;&gt;compute</title>
52 | <path fill="none" stroke="black" d="M120.49,-143.7C129.65,-134.79 140.94,-123.81 150.84,-114.19"/>
53 | <polygon fill="black" stroke="black" points="158.09,-107.14 154.05,-117.34 154.5,-110.63 150.92,-114.12 150.92,-114.12 150.92,-114.12 154.5,-110.63 147.78,-110.89 158.09,-107.14 158.09,-107.14"/>
54 | </g>
55 | <!-- data -->
56 | <g id="node4" class="node">
57 | <title>data</title>
58 | <polygon fill="none" stroke="black" points="300.69,-180 192.69,-180 192.69,-144 300.69,-144 300.69,-180"/>
59 | <text text-anchor="middle" x="246.69" y="-158.3" font-family="Times,serif" font-size="14.00">inputs &amp; outputs</text>
60 | </g>
61 | <!-- data&#45;&gt;compute -->
62 | <g id="edge4" class="edge">
63 | <title>data&#45;&gt;compute</title>
64 | <path fill="none" stroke="black" d="M228.89,-143.7C219.73,-134.79 208.44,-123.81 198.54,-114.19"/>
65 | <polygon fill="black" stroke="black" points="191.3,-107.14 201.6,-110.89 194.88,-110.63 198.47,-114.12 198.47,-114.12 198.47,-114.12 194.88,-110.63 195.33,-117.34 191.3,-107.14 191.3,-107.14"/>
66 | </g>
67 | <!-- solution -->
68 | <g id="node6" class="node">
69 | <title>solution</title>
70 | <polygon fill="none" stroke="black" points="204.69,-36 144.69,-36 144.69,0 204.69,0 204.69,-36"/>
71 | <text text-anchor="middle" x="174.69" y="-14.3" font-family="Times,serif" font-size="14.00">solution</text>
72 | </g>
73 | <!-- compute&#45;&gt;solution -->
74 | <g id="edge5" class="edge">
75 | <title>compute&#45;&gt;solution</title>
76 | <path fill="none" stroke="black" d="M174.69,-71.7C174.69,-63.98 174.69,-54.71 174.69,-46.11"/>
77 | <polygon fill="black" stroke="black" points="174.69,-36.1 179.19,-46.1 174.69,-41.1 174.69,-46.1 174.69,-46.1 174.69,-46.1 174.69,-41.1 170.19,-46.1 174.69,-36.1 174.69,-36.1"/>
78 | </g>
79 | </g>
80 | </svg>
81 | 


--------------------------------------------------------------------------------
/docs/source/images/GraphtikFlowchart-v1.3.0.svg:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8" standalone="no"?>
  2 | <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
  3 |  "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
  4 | <!-- Generated by graphviz version 2.43.0 (0)
  5 |  -->
  6 | <!-- Title: %3 Pages: 1 -->
  7 | <svg width="445pt" height="502pt"
  8 |  viewBox="0.00 0.00 445.17 502.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
  9 | <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 498)">
 10 | <title>%3</title>
 11 | <polygon fill="white" stroke="transparent" points="-4,4 -4,-498 441.17,-498 441.17,4 -4,4"/>
 12 | <text text-anchor="middle" x="218.59" y="-478.8" font-family="Times,serif" font-size="14.00">graphkit&#45;v1.3.0 flowchart</text>
 13 | <g id="clust1" class="cluster">
 14 | <title>cluster_compute</title>
 15 | <polygon fill="none" stroke="black" stroke-dasharray="5,2" points="95.67,-44 95.67,-263 343.67,-263 343.67,-44 95.67,-44"/>
 16 | <text text-anchor="middle" x="219.67" y="-51.8" font-family="bold" font-size="14.00">compute</text>
 17 | </g>
 18 | <!-- operations -->
 19 | <g id="node1" class="node">
 20 | <title>operations</title>
 21 | <polygon fill="none" stroke="black" points="175.51,-471 35.78,-471 -0.17,-435 139.56,-435 175.51,-471"/>
 22 | <text text-anchor="middle" x="87.67" y="-449.3" font-family="italic" font-size="14.00">operations</text>
 23 | </g>
 24 | <!-- compose -->
 25 | <g id="node2" class="node">
 26 | <title>compose</title>
 27 | <ellipse fill="none" stroke="black" cx="87.67" cy="-381" rx="51.99" ry="18"/>
 28 | <text text-anchor="middle" x="87.67" y="-377.3" font-family="italic" font-size="14.00">compose</text>
 29 | </g>
 30 | <!-- operations&#45;&gt;compose -->
 31 | <g id="edge1" class="edge">
 32 | <title>operations&#45;&gt;compose</title>
 33 | <path fill="none" stroke="black" d="M87.67,-434.7C87.67,-426.98 87.67,-417.71 87.67,-409.11"/>
 34 | <polygon fill="black" stroke="black" points="87.67,-399.1 92.17,-409.1 87.67,-404.1 87.67,-409.1 87.67,-409.1 87.67,-409.1 87.67,-404.1 83.17,-409.1 87.67,-399.1 87.67,-399.1"/>
 35 | </g>
 36 | <!-- network -->
 37 | <g id="node3" class="node">
 38 | <title>network</title>
 39 | <polygon fill="none" stroke="black" points="160.02,-327 44.93,-327 15.32,-291 130.41,-291 160.02,-327"/>
 40 | <text text-anchor="middle" x="87.67" y="-305.3" font-family="italic" font-size="14.00">network</text>
 41 | </g>
 42 | <!-- compose&#45;&gt;network -->
 43 | <g id="edge2" class="edge">
 44 | <title>compose&#45;&gt;network</title>
 45 | <path fill="none" stroke="black" d="M87.67,-362.7C87.67,-354.98 87.67,-345.71 87.67,-337.11"/>
 46 | <polygon fill="black" stroke="black" points="87.67,-327.1 92.17,-337.1 87.67,-332.1 87.67,-337.1 87.67,-337.1 87.67,-337.1 87.67,-332.1 83.17,-337.1 87.67,-327.1 87.67,-327.1"/>
 47 | </g>
 48 | <!-- compile -->
 49 | <g id="node6" class="node">
 50 | <title>compile</title>
 51 | <ellipse fill="none" stroke="black" cx="219.67" cy="-237" rx="47.39" ry="18"/>
 52 | <text text-anchor="middle" x="219.67" y="-233.3" font-family="italic" font-size="14.00">compile</text>
 53 | </g>
 54 | <!-- network&#45;&gt;compile -->
 55 | <g id="edge3" class="edge">
 56 | <title>network&#45;&gt;compile</title>
 57 | <path fill="none" stroke="black" d="M119.96,-290.88C139.33,-280.6 164,-267.52 183.95,-256.94"/>
 58 | <polygon fill="black" stroke="black" points="193.04,-252.12 186.31,-260.78 188.62,-254.47 184.2,-256.81 184.2,-256.81 184.2,-256.81 188.62,-254.47 182.1,-252.83 193.04,-252.12 193.04,-252.12"/>
 59 | </g>
 60 | <!-- inputs -->
 61 | <g id="node4" class="node">
 62 | <title>inputs</title>
 63 | <polygon fill="none" stroke="black" points="261.17,-327 178.17,-327 178.17,-291 261.17,-291 261.17,-327"/>
 64 | <text text-anchor="middle" x="219.67" y="-305.3" font-family="Times,serif" font-size="14.00">input names</text>
 65 | </g>
 66 | <!-- inputs&#45;&gt;compile -->
 67 | <g id="edge4" class="edge">
 68 | <title>inputs&#45;&gt;compile</title>
 69 | <path fill="none" stroke="black" d="M219.67,-290.7C219.67,-282.98 219.67,-273.71 219.67,-265.11"/>
 70 | <polygon fill="black" stroke="black" points="219.67,-255.1 224.17,-265.1 219.67,-260.1 219.67,-265.1 219.67,-265.1 219.67,-265.1 219.67,-260.1 215.17,-265.1 219.67,-255.1 219.67,-255.1"/>
 71 | </g>
 72 | <!-- outputs -->
 73 | <g id="node5" class="node">
 74 | <title>outputs</title>
 75 | <polygon fill="none" stroke="black" points="369.67,-327 279.67,-327 279.67,-291 369.67,-291 369.67,-327"/>
 76 | <text text-anchor="middle" x="324.67" y="-305.3" font-family="Times,serif" font-size="14.00">output names</text>
 77 | </g>
 78 | <!-- outputs&#45;&gt;compile -->
 79 | <g id="edge5" class="edge">
 80 | <title>outputs&#45;&gt;compile</title>
 81 | <path fill="none" stroke="black" d="M298.98,-290.88C284.47,-281.2 266.22,-269.03 250.88,-258.81"/>
 82 | <polygon fill="black" stroke="black" points="242.24,-253.05 253.06,-254.85 246.4,-255.82 250.56,-258.59 250.56,-258.59 250.56,-258.59 246.4,-255.82 248.06,-262.34 242.24,-253.05 242.24,-253.05"/>
 83 | </g>
 84 | <!-- plan -->
 85 | <g id="node7" class="node">
 86 | <title>plan</title>
 87 | <polygon fill="none" stroke="black" points="335.69,-183 151.13,-183 103.65,-147 288.21,-147 335.69,-183"/>
 88 | <text text-anchor="middle" x="219.67" y="-161.3" font-family="italic" font-size="14.00">execution plan</text>
 89 | </g>
 90 | <!-- compile&#45;&gt;plan -->
 91 | <g id="edge6" class="edge">
 92 | <title>compile&#45;&gt;plan</title>
 93 | <path fill="none" stroke="black" d="M219.67,-218.7C219.67,-210.98 219.67,-201.71 219.67,-193.11"/>
 94 | <polygon fill="black" stroke="black" points="219.67,-183.1 224.17,-193.1 219.67,-188.1 219.67,-193.1 219.67,-193.1 219.67,-193.1 219.67,-188.1 215.17,-193.1 219.67,-183.1 219.67,-183.1"/>
 95 | </g>
 96 | <!-- execute -->
 97 | <g id="node8" class="node">
 98 | <title>execute</title>
 99 | <ellipse fill="none" stroke="black" cx="253.67" cy="-93" rx="47.39" ry="18"/>
100 | <text text-anchor="middle" x="253.67" y="-89.3" font-family="italic" font-size="14.00">execute</text>
101 | </g>
102 | <!-- plan&#45;&gt;execute -->
103 | <g id="edge7" class="edge">
104 | <title>plan&#45;&gt;execute</title>
105 | <path fill="none" stroke="black" d="M228.07,-146.7C231.95,-138.73 236.62,-129.1 240.91,-120.26"/>
106 | <polygon fill="black" stroke="black" points="245.36,-111.1 245.04,-122.07 243.18,-115.6 240.99,-120.1 240.99,-120.1 240.99,-120.1 243.18,-115.6 236.95,-118.13 245.36,-111.1 245.36,-111.1"/>
107 | </g>
108 | <!-- solution -->
109 | <g id="node10" class="node">
110 | <title>solution</title>
111 | <polygon fill="none" stroke="black" points="241.67,-36 181.67,-36 181.67,0 241.67,0 241.67,-36"/>
112 | <text text-anchor="middle" x="211.67" y="-14.3" font-family="Times,serif" font-size="14.00">solution</text>
113 | </g>
114 | <!-- execute&#45;&gt;solution -->
115 | <g id="edge9" class="edge">
116 | <title>execute&#45;&gt;solution</title>
117 | <path fill="none" stroke="black" d="M243.93,-75.07C238.77,-66.1 232.33,-54.9 226.56,-44.87"/>
118 | <polygon fill="black" stroke="black" points="221.5,-36.09 230.39,-42.52 224,-40.43 226.49,-44.76 226.49,-44.76 226.49,-44.76 224,-40.43 222.59,-47 221.5,-36.09 221.5,-36.09"/>
119 | </g>
120 | <!-- overwrites -->
121 | <g id="node11" class="node">
122 | <title>overwrites</title>
123 | <polygon fill="none" stroke="black" points="333.67,-36 259.67,-36 259.67,0 333.67,0 333.67,-36"/>
124 | <text text-anchor="middle" x="296.67" y="-14.3" font-family="Times,serif" font-size="14.00">overwrites</text>
125 | </g>
126 | <!-- execute&#45;&gt;overwrites -->
127 | <g id="edge10" class="edge">
128 | <title>execute&#45;&gt;overwrites</title>
129 | <path fill="none" stroke="black" d="M263.64,-75.07C268.92,-66.1 275.52,-54.9 281.43,-44.87"/>
130 | <polygon fill="black" stroke="black" points="286.6,-36.09 285.4,-46.99 284.06,-40.4 281.53,-44.71 281.53,-44.71 281.53,-44.71 284.06,-40.4 277.65,-42.42 286.6,-36.09 286.6,-36.09"/>
131 | </g>
132 | <!-- values -->
133 | <g id="node9" class="node">
134 | <title>values</title>
135 | <polygon fill="none" stroke="black" points="437.17,-183 354.17,-183 354.17,-147 437.17,-147 437.17,-183"/>
136 | <text text-anchor="middle" x="395.67" y="-161.3" font-family="Times,serif" font-size="14.00">input values</text>
137 | </g>
138 | <!-- values&#45;&gt;execute -->
139 | <g id="edge8" class="edge">
140 | <title>values&#45;&gt;execute</title>
141 | <path fill="none" stroke="black" d="M360.93,-146.88C339.62,-136.37 312.35,-122.92 290.63,-112.22"/>
142 | <polygon fill="black" stroke="black" points="281.4,-107.67 292.36,-108.05 285.88,-109.88 290.37,-112.09 290.37,-112.09 290.37,-112.09 285.88,-109.88 288.38,-116.13 281.4,-107.67 281.4,-107.67"/>
143 | </g>
144 | </g>
145 | </svg>
146 | 


--------------------------------------------------------------------------------
/docs/source/images/GraphtikFlowchart-v4.1.0.svg:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8" standalone="no"?>
  2 | <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
  3 |  "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
  4 | <!-- Generated by graphviz version 2.43.0 (0)
  5 |  -->
  6 | <!-- Title: %3 Pages: 1 -->
  7 | <svg width="497pt" height="502pt"
  8 |  viewBox="0.00 0.00 497.17 502.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
  9 | <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 498)">
 10 | <title>%3</title>
 11 | <polygon fill="white" stroke="transparent" points="-4,4 -4,-498 493.17,-498 493.17,4 -4,4"/>
 12 | <text text-anchor="middle" x="244.59" y="-478.8" font-family="Times,serif" font-size="14.00">graphtik&#45;v4.1.0 flowchart</text>
 13 | <g id="clust1" class="cluster">
 14 | <title>cluster_compute</title>
 15 | <g id="a_clust1"><a xlink:href="arch.html#term-compute" xlink:title="process, NetOp&#39;s method">
 16 | <polygon fill="none" stroke="black" stroke-dasharray="5,2" points="147.67,-44 147.67,-263 395.67,-263 395.67,-44 147.67,-44"/>
 17 | <text text-anchor="middle" x="271.67" y="-51.8" font-family="bold" font-size="14.00">compute</text>
 18 | </a>
 19 | </g>
 20 | </g>
 21 | <!-- operations -->
 22 | <g id="node1" class="node">
 23 | <title>operations</title>
 24 | <g id="a_node1"><a xlink:href="arch.html#term-operation" xlink:title="class">
 25 | <polygon fill="none" stroke="black" points="175.51,-471 35.78,-471 -0.17,-435 139.56,-435 175.51,-471"/>
 26 | <text text-anchor="middle" x="87.67" y="-449.3" font-family="italic" font-size="14.00">operations</text>
 27 | </a>
 28 | </g>
 29 | </g>
 30 | <!-- compose -->
 31 | <g id="node2" class="node">
 32 | <title>compose</title>
 33 | <g id="a_node2"><a xlink:href="arch.html#term-composition" xlink:title="phase">
 34 | <ellipse fill="none" stroke="black" cx="87.67" cy="-381" rx="51.99" ry="18"/>
 35 | <text text-anchor="middle" x="87.67" y="-377.3" font-family="italic" font-size="14.00">compose</text>
 36 | </a>
 37 | </g>
 38 | </g>
 39 | <!-- operations&#45;&gt;compose -->
 40 | <g id="edge1" class="edge">
 41 | <title>operations&#45;&gt;compose</title>
 42 | <path fill="none" stroke="black" d="M87.67,-434.7C87.67,-426.98 87.67,-417.71 87.67,-409.11"/>
 43 | <polygon fill="black" stroke="black" points="87.67,-399.1 92.17,-409.1 87.67,-404.1 87.67,-409.1 87.67,-409.1 87.67,-409.1 87.67,-404.1 83.17,-409.1 87.67,-399.1 87.67,-399.1"/>
 44 | </g>
 45 | <!-- network -->
 46 | <g id="node3" class="node">
 47 | <title>network</title>
 48 | <g id="a_node3"><a xlink:href="arch.html#term-network" xlink:title="class">
 49 | <polygon fill="none" stroke="black" points="160.02,-327 44.93,-327 15.32,-291 130.41,-291 160.02,-327"/>
 50 | <text text-anchor="middle" x="87.67" y="-305.3" font-family="italic" font-size="14.00">network</text>
 51 | </a>
 52 | </g>
 53 | </g>
 54 | <!-- compose&#45;&gt;network -->
 55 | <g id="edge2" class="edge">
 56 | <title>compose&#45;&gt;network</title>
 57 | <path fill="none" stroke="black" d="M87.67,-362.7C87.67,-354.98 87.67,-345.71 87.67,-337.11"/>
 58 | <polygon fill="black" stroke="black" points="87.67,-327.1 92.17,-337.1 87.67,-332.1 87.67,-337.1 87.67,-337.1 87.67,-337.1 87.67,-332.1 83.17,-337.1 87.67,-327.1 87.67,-327.1"/>
 59 | </g>
 60 | <!-- compile -->
 61 | <g id="node7" class="node">
 62 | <title>compile</title>
 63 | <g id="a_node7"><a xlink:href="arch.html#term-compilation" xlink:title="phase, Network&#39;s method">
 64 | <ellipse fill="none" stroke="black" cx="271.67" cy="-237" rx="47.39" ry="18"/>
 65 | <text text-anchor="middle" x="271.67" y="-233.3" font-family="italic" font-size="14.00">compile</text>
 66 | </a>
 67 | </g>
 68 | </g>
 69 | <!-- network&#45;&gt;compile -->
 70 | <g id="edge3" class="edge">
 71 | <title>network&#45;&gt;compile</title>
 72 | <path fill="none" stroke="black" d="M131.28,-291.41C161.07,-280.08 200.32,-265.14 229.74,-253.95"/>
 73 | <polygon fill="black" stroke="black" points="239.32,-250.31 231.57,-258.07 234.64,-252.09 229.97,-253.86 229.97,-253.86 229.97,-253.86 234.64,-252.09 228.37,-249.66 239.32,-250.31 239.32,-250.31"/>
 74 | </g>
 75 | <!-- inputs -->
 76 | <g id="node4" class="node">
 77 | <title>inputs</title>
 78 | <g id="a_node4"><a xlink:href="arch.html#term-inputs" xlink:title="mappings">
 79 | <polygon fill="none" stroke="black" points="261.17,-327 178.17,-327 178.17,-291 261.17,-291 261.17,-327"/>
 80 | <text text-anchor="middle" x="219.67" y="-305.3" font-family="Times,serif" font-size="14.00">input names</text>
 81 | </a>
 82 | </g>
 83 | </g>
 84 | <!-- inputs&#45;&gt;compile -->
 85 | <g id="edge4" class="edge">
 86 | <title>inputs&#45;&gt;compile</title>
 87 | <path fill="none" stroke="black" d="M232.52,-290.7C238.79,-282.27 246.43,-271.97 253.3,-262.73"/>
 88 | <polygon fill="black" stroke="black" points="259.32,-254.62 256.97,-265.33 256.34,-258.64 253.36,-262.65 253.36,-262.65 253.36,-262.65 256.34,-258.64 249.75,-259.97 259.32,-254.62 259.32,-254.62"/>
 89 | </g>
 90 | <!-- outputs -->
 91 | <g id="node5" class="node">
 92 | <title>outputs</title>
 93 | <g id="a_node5"><a xlink:href="arch.html#term-outputs" xlink:title="mappings">
 94 | <polygon fill="none" stroke="black" points="369.67,-327 279.67,-327 279.67,-291 369.67,-291 369.67,-327"/>
 95 | <text text-anchor="middle" x="324.67" y="-305.3" font-family="Times,serif" font-size="14.00">output names</text>
 96 | </a>
 97 | </g>
 98 | </g>
 99 | <!-- outputs&#45;&gt;compile -->
100 | <g id="edge5" class="edge">
101 | <title>outputs&#45;&gt;compile</title>
102 | <path fill="none" stroke="black" d="M311.57,-290.7C305.19,-282.27 297.39,-271.97 290.4,-262.73"/>
103 | <polygon fill="black" stroke="black" points="284.26,-254.62 293.88,-259.88 287.27,-258.61 290.29,-262.59 290.29,-262.59 290.29,-262.59 287.27,-258.61 286.7,-265.31 284.26,-254.62 284.26,-254.62"/>
104 | </g>
105 | <!-- predicate -->
106 | <g id="node6" class="node">
107 | <title>predicate</title>
108 | <g id="a_node6"><a xlink:href="arch.html#term-node-predicate" xlink:title="function">
109 | <polygon fill="none" stroke="black" points="483.67,-327 387.67,-327 387.67,-291 483.67,-291 483.67,-327"/>
110 | <text text-anchor="middle" x="435.67" y="-305.3" font-family="Times,serif" font-size="14.00">node predicate</text>
111 | </a>
112 | </g>
113 | </g>
114 | <!-- predicate&#45;&gt;compile -->
115 | <g id="edge6" class="edge">
116 | <title>predicate&#45;&gt;compile</title>
117 | <path fill="none" stroke="black" d="M395.55,-290.88C369.91,-279.93 336.79,-265.79 311.25,-254.89"/>
118 | <polygon fill="black" stroke="black" points="301.86,-250.89 312.83,-250.67 306.46,-252.85 311.06,-254.81 311.06,-254.81 311.06,-254.81 306.46,-252.85 309.29,-258.95 301.86,-250.89 301.86,-250.89"/>
119 | </g>
120 | <!-- plan -->
121 | <g id="node8" class="node">
122 | <title>plan</title>
123 | <g id="a_node8"><a xlink:href="arch.html#term-execution-plan" xlink:title="class">
124 | <polygon fill="none" stroke="black" points="387.69,-183 203.13,-183 155.65,-147 340.21,-147 387.69,-183"/>
125 | <text text-anchor="middle" x="271.67" y="-161.3" font-family="italic" font-size="14.00">execution plan</text>
126 | </a>
127 | </g>
128 | </g>
129 | <!-- compile&#45;&gt;plan -->
130 | <g id="edge7" class="edge">
131 | <title>compile&#45;&gt;plan</title>
132 | <path fill="none" stroke="black" d="M271.67,-218.7C271.67,-210.98 271.67,-201.71 271.67,-193.11"/>
133 | <polygon fill="black" stroke="black" points="271.67,-183.1 276.17,-193.1 271.67,-188.1 271.67,-193.1 271.67,-193.1 271.67,-193.1 271.67,-188.1 267.17,-193.1 271.67,-183.1 271.67,-183.1"/>
134 | </g>
135 | <!-- execute -->
136 | <g id="node9" class="node">
137 | <title>execute</title>
138 | <g id="a_node9"><a xlink:href="arch.html#term-execution" xlink:title="phase, Plan&#39;s method">
139 | <ellipse fill="none" stroke="black" cx="305.67" cy="-93" rx="47.39" ry="18"/>
140 | <text text-anchor="middle" x="305.67" y="-89.3" font-family="italic" font-size="14.00">execute</text>
141 | </a>
142 | </g>
143 | </g>
144 | <!-- plan&#45;&gt;execute -->
145 | <g id="edge8" class="edge">
146 | <title>plan&#45;&gt;execute</title>
147 | <path fill="none" stroke="black" d="M280.07,-146.7C283.95,-138.73 288.62,-129.1 292.91,-120.26"/>
148 | <polygon fill="black" stroke="black" points="297.36,-111.1 297.04,-122.07 295.18,-115.6 292.99,-120.1 292.99,-120.1 292.99,-120.1 295.18,-115.6 288.95,-118.13 297.36,-111.1 297.36,-111.1"/>
149 | </g>
150 | <!-- solution -->
151 | <g id="node11" class="node">
152 | <title>solution</title>
153 | <g id="a_node11"><a xlink:href="arch.html#term-solution" xlink:title="class">
154 | <polygon fill="none" stroke="black" points="363.02,-36 271.79,-36 248.32,0 339.55,0 363.02,-36"/>
155 | <text text-anchor="middle" x="305.67" y="-14.3" font-family="Times,serif" font-size="14.00">solution</text>
156 | </a>
157 | </g>
158 | </g>
159 | <!-- execute&#45;&gt;solution -->
160 | <g id="edge10" class="edge">
161 | <title>execute&#45;&gt;solution</title>
162 | <path fill="none" stroke="black" d="M305.67,-74.7C305.67,-66.25 305.67,-55.87 305.67,-46.37"/>
163 | <polygon fill="black" stroke="black" points="305.67,-36.18 310.17,-46.18 305.67,-41.18 305.67,-46.18 305.67,-46.18 305.67,-46.18 305.67,-41.18 301.17,-46.18 305.67,-36.18 305.67,-36.18"/>
164 | </g>
165 | <!-- values -->
166 | <g id="node10" class="node">
167 | <title>values</title>
168 | <g id="a_node10"><a xlink:href="arch.html#term-inputs" xlink:title="mappings">
169 | <polygon fill="none" stroke="black" points="489.17,-183 406.17,-183 406.17,-147 489.17,-147 489.17,-183"/>
170 | <text text-anchor="middle" x="447.67" y="-161.3" font-family="Times,serif" font-size="14.00">input values</text>
171 | </a>
172 | </g>
173 | </g>
174 | <!-- values&#45;&gt;execute -->
175 | <g id="edge9" class="edge">
176 | <title>values&#45;&gt;execute</title>
177 | <path fill="none" stroke="black" d="M412.93,-146.88C391.62,-136.37 364.35,-122.92 342.63,-112.22"/>
178 | <polygon fill="black" stroke="black" points="333.4,-107.67 344.36,-108.05 337.88,-109.88 342.37,-112.09 342.37,-112.09 342.37,-112.09 337.88,-109.88 340.38,-116.13 333.4,-107.67 333.4,-107.67"/>
179 | </g>
180 | </g>
181 | </svg>
182 | 


--------------------------------------------------------------------------------
/docs/source/images/barebone_3ops.svg:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8" standalone="no"?>
  2 | <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
  3 |  "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
  4 | <!-- Generated by graphviz version 2.43.0 (0)
  5 |  -->
  6 | <!-- Title: G Pages: 1 -->
  7 | <svg width="154pt" height="499pt"
  8 |  viewBox="0.00 0.00 154.00 499.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
  9 | <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 495)">
 10 | <title>G</title>
 11 | <polygon fill="white" stroke="transparent" points="-4,4 -4,-495 150,-495 150,4 -4,4"/>
 12 | <text text-anchor="middle" x="73" y="-7.8" font-family="italic" font-size="14.00">graphop</text>
 13 | <!-- a -->
 14 | <g id="node1" class="node">
 15 | <title>a</title>
 16 | <polygon fill="none" stroke="black" points="131,-491 77,-491 77,-455 131,-455 131,-491"/>
 17 | <text text-anchor="middle" x="104" y="-469.3" font-family="Times,serif" font-size="14.00">a</text>
 18 | </g>
 19 | <!-- mul1 -->
 20 | <g id="node2" class="node">
 21 | <title>mul1</title>
 22 | <ellipse fill="none" stroke="black" cx="42" cy="-401" rx="34.39" ry="18"/>
 23 | <text text-anchor="middle" x="42" y="-397.3" font-family="italic" font-size="14.00">mul1</text>
 24 | </g>
 25 | <!-- a&#45;&gt;mul1 -->
 26 | <g id="edge1" class="edge">
 27 | <title>a&#45;&gt;mul1</title>
 28 | <path fill="none" stroke="black" d="M88.67,-454.7C80.83,-445.84 71.17,-434.93 62.67,-425.34"/>
 29 | <polygon fill="black" stroke="black" points="65.13,-422.84 55.88,-417.67 59.89,-427.48 65.13,-422.84"/>
 30 | </g>
 31 | <!-- sub1 -->
 32 | <g id="node5" class="node">
 33 | <title>sub1</title>
 34 | <ellipse fill="none" stroke="black" cx="73" cy="-257" rx="33.29" ry="18"/>
 35 | <text text-anchor="middle" x="73" y="-253.3" font-family="italic" font-size="14.00">sub1</text>
 36 | </g>
 37 | <!-- a&#45;&gt;sub1 -->
 38 | <g id="edge2" class="edge">
 39 | <title>a&#45;&gt;sub1</title>
 40 | <path fill="none" stroke="black" d="M101.81,-454.58C98.04,-425.02 89.99,-363.15 82,-311 80.71,-302.57 79.19,-293.4 77.78,-285.1"/>
 41 | <polygon fill="black" stroke="black" points="81.21,-284.42 76.06,-275.16 74.31,-285.61 81.21,-284.42"/>
 42 | </g>
 43 | <!-- ab -->
 44 | <g id="node4" class="node">
 45 | <title>ab</title>
 46 | <polygon fill="none" stroke="black" points="73,-347 19,-347 19,-311 73,-311 73,-347"/>
 47 | <text text-anchor="middle" x="46" y="-325.3" font-family="Times,serif" font-size="14.00">ab</text>
 48 | </g>
 49 | <!-- mul1&#45;&gt;ab -->
 50 | <g id="edge3" class="edge">
 51 | <title>mul1&#45;&gt;ab</title>
 52 | <path fill="none" stroke="black" d="M42.99,-382.7C43.43,-374.98 43.96,-365.71 44.45,-357.11"/>
 53 | <polygon fill="black" stroke="black" points="47.95,-357.29 45.02,-347.1 40.96,-356.89 47.95,-357.29"/>
 54 | </g>
 55 | <!-- b -->
 56 | <g id="node3" class="node">
 57 | <title>b</title>
 58 | <polygon fill="none" stroke="black" points="59,-491 5,-491 5,-455 59,-455 59,-491"/>
 59 | <text text-anchor="middle" x="32" y="-469.3" font-family="Times,serif" font-size="14.00">b</text>
 60 | </g>
 61 | <!-- b&#45;&gt;mul1 -->
 62 | <g id="edge4" class="edge">
 63 | <title>b&#45;&gt;mul1</title>
 64 | <path fill="none" stroke="black" d="M34.47,-454.7C35.57,-446.98 36.9,-437.71 38.13,-429.11"/>
 65 | <polygon fill="black" stroke="black" points="41.61,-429.5 39.56,-419.1 34.68,-428.51 41.61,-429.5"/>
 66 | </g>
 67 | <!-- ab&#45;&gt;sub1 -->
 68 | <g id="edge5" class="edge">
 69 | <title>ab&#45;&gt;sub1</title>
 70 | <path fill="none" stroke="black" d="M52.67,-310.7C55.71,-302.81 59.38,-293.3 62.76,-284.55"/>
 71 | <polygon fill="black" stroke="black" points="66.07,-285.69 66.4,-275.1 59.54,-283.17 66.07,-285.69"/>
 72 | </g>
 73 | <!-- a_minus_ab -->
 74 | <g id="node6" class="node">
 75 | <title>a_minus_ab</title>
 76 | <polygon fill="none" stroke="black" points="114.5,-203 31.5,-203 31.5,-167 114.5,-167 114.5,-203"/>
 77 | <text text-anchor="middle" x="73" y="-181.3" font-family="Times,serif" font-size="14.00">a_minus_ab</text>
 78 | </g>
 79 | <!-- sub1&#45;&gt;a_minus_ab -->
 80 | <g id="edge6" class="edge">
 81 | <title>sub1&#45;&gt;a_minus_ab</title>
 82 | <path fill="none" stroke="black" d="M73,-238.7C73,-230.98 73,-221.71 73,-213.11"/>
 83 | <polygon fill="black" stroke="black" points="76.5,-213.1 73,-203.1 69.5,-213.1 76.5,-213.1"/>
 84 | </g>
 85 | <!-- abspow1 -->
 86 | <g id="node7" class="node">
 87 | <title>abspow1</title>
 88 | <ellipse fill="none" stroke="black" cx="73" cy="-113" rx="51.99" ry="18"/>
 89 | <text text-anchor="middle" x="73" y="-109.3" font-family="italic" font-size="14.00">abspow1</text>
 90 | </g>
 91 | <!-- a_minus_ab&#45;&gt;abspow1 -->
 92 | <g id="edge7" class="edge">
 93 | <title>a_minus_ab&#45;&gt;abspow1</title>
 94 | <path fill="none" stroke="black" d="M73,-166.7C73,-158.98 73,-149.71 73,-141.11"/>
 95 | <polygon fill="black" stroke="black" points="76.5,-141.1 73,-131.1 69.5,-141.1 76.5,-141.1"/>
 96 | </g>
 97 | <!-- abs_a_minus_ab_cubed -->
 98 | <g id="node8" class="node">
 99 | <title>abs_a_minus_ab_cubed</title>
100 | <polygon fill="none" stroke="black" points="146,-59 0,-59 0,-23 146,-23 146,-59"/>
101 | <text text-anchor="middle" x="73" y="-37.3" font-family="Times,serif" font-size="14.00">abs_a_minus_ab_cubed</text>
102 | </g>
103 | <!-- abspow1&#45;&gt;abs_a_minus_ab_cubed -->
104 | <g id="edge8" class="edge">
105 | <title>abspow1&#45;&gt;abs_a_minus_ab_cubed</title>
106 | <path fill="none" stroke="black" d="M73,-94.7C73,-86.98 73,-77.71 73,-69.11"/>
107 | <polygon fill="black" stroke="black" points="76.5,-69.1 73,-59.1 69.5,-69.1 76.5,-69.1"/>
108 | </g>
109 | </g>
110 | </svg>
111 | 


--------------------------------------------------------------------------------
/docs/source/images/unpruned_useless_provides.svg:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8" standalone="no"?>
  2 | <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
  3 |  "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
  4 | <!-- Generated by graphviz version 2.43.0 (0)
  5 |  -->
  6 | <!-- Title: G Pages: 1 -->
  7 | <svg width="476pt" height="598pt"
  8 |  viewBox="0.00 0.00 476.00 598.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
  9 | <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 594)">
 10 | <title>G</title>
 11 | <polygon fill="white" stroke="transparent" points="-4,4 -4,-594 472,-594 472,4 -4,4"/>
 12 | <text text-anchor="middle" x="234" y="-7.8" font-family="italic" font-size="14.00">bigger_graph</text>
 13 | <g id="clust1" class="cluster">
 14 | <title>cluster_after prunning</title>
 15 | <polygon fill="none" stroke="black" points="8,-31 8,-582 242,-582 242,-31 8,-31"/>
 16 | <text text-anchor="middle" x="125" y="-566.8" font-family="italic" font-size="14.00">after prunning</text>
 17 | </g>
 18 | <!-- ab -->
 19 | <g id="node1" class="node">
 20 | <title>ab</title>
 21 | <polygon fill="wheat" stroke="black" points="304,-313 250,-313 250,-277 304,-277 304,-313"/>
 22 | <text text-anchor="middle" x="277" y="-291.3" font-family="Times,serif" font-size="14.00">ab</text>
 23 | </g>
 24 | <!-- abs_a_minus_ab_cubed -->
 25 | <g id="node2" class="node">
 26 | <title>abs_a_minus_ab_cubed</title>
 27 | <polygon fill="wheat" stroke="black" points="468,-313 322,-313 322,-277 468,-277 468,-313"/>
 28 | <text text-anchor="middle" x="395" y="-291.3" font-family="Times,serif" font-size="14.00">abs_a_minus_ab_cubed</text>
 29 | </g>
 30 | <!-- a -->
 31 | <g id="node3" class="node">
 32 | <title>a</title>
 33 | <polygon fill="none" stroke="#990000" stroke-width="3" points="133,-527.44 160,-515 187,-527.44 186.97,-547.56 133.03,-547.56 133,-527.44"/>
 34 | <text text-anchor="middle" x="160" y="-529.3" font-family="Times,serif" font-size="14.00">a</text>
 35 | </g>
 36 | <!-- graphop -->
 37 | <g id="node4" class="node">
 38 | <title>graphop</title>
 39 | <polygon fill="none" stroke="black" points="185.88,-350.05 189.11,-350.15 192.3,-350.3 195.45,-350.49 198.54,-350.74 201.56,-351.03 204.5,-351.36 207.34,-351.75 210.07,-352.18 212.69,-352.65 215.18,-353.16 217.54,-353.71 219.76,-354.31 221.83,-354.94 223.74,-355.61 225.49,-356.31 227.08,-357.04 228.5,-357.8 229.75,-358.59 230.83,-359.41 231.74,-360.25 232.47,-361.11 233.03,-361.99 233.42,-362.89 233.65,-363.8 233.71,-364.72 233.61,-365.65 233.35,-366.59 232.95,-367.53 232.4,-368.47 231.71,-369.41 230.89,-370.35 229.94,-371.28 228.88,-372.2 227.7,-373.11 226.42,-374.01 225.04,-374.89 223.57,-375.75 222.02,-376.59 220.39,-377.41 218.69,-378.2 216.93,-378.96 215.12,-379.69 213.25,-380.39 211.33,-381.06 209.38,-381.69 207.39,-382.29 205.37,-382.84 203.33,-383.35 201.26,-383.82 199.17,-384.25 197.07,-384.64 194.95,-384.97 192.82,-385.26 190.68,-385.51 188.54,-385.7 186.39,-385.85 184.23,-385.95 182.08,-386 179.92,-386 177.77,-385.95 175.61,-385.85 173.46,-385.7 171.32,-385.51 169.18,-385.26 167.05,-384.97 164.93,-384.64 162.83,-384.25 160.74,-383.82 158.67,-383.35 156.63,-382.84 154.61,-382.29 152.62,-381.69 150.67,-381.06 148.75,-380.39 146.88,-379.69 145.07,-378.96 143.31,-378.2 141.61,-377.41 139.98,-376.59 138.43,-375.75 136.96,-374.89 135.58,-374.01 134.3,-373.11 133.12,-372.2 132.06,-371.28 131.11,-370.35 130.29,-369.41 129.6,-368.47 129.05,-367.53 128.65,-366.59 128.39,-365.65 128.29,-364.72 128.35,-363.8 128.58,-362.89 128.97,-361.99 129.53,-361.11 130.26,-360.25 131.17,-359.41 132.25,-358.59 133.5,-357.8 134.92,-357.04 136.51,-356.31 138.26,-355.61 140.17,-354.94 142.24,-354.31 144.46,-353.71 146.82,-353.16 149.31,-352.65 151.93,-352.18 154.66,-351.75 157.5,-351.36 160.44,-351.03 163.46,-350.74 166.55,-350.49 169.7,-350.3 172.89,-350.15 176.12,-350.05 179.37,-350 182.63,-350 185.88,-350.05"/>
 40 | <text text-anchor="middle" x="181" y="-364.3" font-family="italic" font-size="14.00">graphop</text>
 41 | </g>
 42 | <!-- a&#45;&gt;graphop -->
 43 | <g id="edge1" class="edge">
 44 | <title>a&#45;&gt;graphop</title>
 45 | <path fill="none" stroke="black" stroke-dasharray="5,2" d="M164.41,-516.77C168.45,-502.11 174.22,-479.24 177,-459 179.89,-437.99 180.84,-413.98 181.09,-396.05"/>
 46 | <polygon fill="black" stroke="black" points="184.59,-396.03 181.17,-386 177.59,-395.98 184.59,-396.03"/>
 47 | </g>
 48 | <!-- b -->
 49 | <g id="node5" class="node">
 50 | <title>b</title>
 51 | <polygon fill="none" stroke="#990000" stroke-width="3" points="70,-435.44 97,-423 124,-435.44 123.97,-455.56 70.03,-455.56 70,-435.44"/>
 52 | <text text-anchor="middle" x="97" y="-437.3" font-family="Times,serif" font-size="14.00">b</text>
 53 | </g>
 54 | <!-- a&#45;&gt;b -->
 55 | <g id="edge10" class="edge">
 56 | <title>a&#45;&gt;b</title>
 57 | <path fill="none" stroke="#009999" stroke-width="3" stroke-dasharray="1,5" d="M150.97,-519.11C140.95,-504.78 124.59,-481.41 112.5,-464.15"/>
 58 | <polygon fill="#009999" stroke="#009999" stroke-width="3" points="106.63,-455.76 116.05,-461.37 110.32,-459.28 113.18,-463.38 112.36,-463.95 111.54,-464.52 108.68,-460.43 108.68,-466.53 106.63,-455.76 106.63,-455.76"/>
 59 | <text text-anchor="middle" x="140.5" y="-482.6" font-family="bold" font-size="18.00" fill="#009999">2</text>
 60 | </g>
 61 | <!-- graphop&#45;&gt;ab -->
 62 | <g id="edge2" class="edge">
 63 | <title>graphop&#45;&gt;ab</title>
 64 | <path fill="none" stroke="black" d="M202.78,-350.89C215.43,-341.54 231.57,-329.6 245.57,-319.24"/>
 65 | <polygon fill="black" stroke="black" points="248,-321.8 253.96,-313.04 243.84,-316.17 248,-321.8"/>
 66 | </g>
 67 | <!-- graphop&#45;&gt;abs_a_minus_ab_cubed -->
 68 | <g id="edge4" class="edge">
 69 | <title>graphop&#45;&gt;abs_a_minus_ab_cubed</title>
 70 | <path fill="none" stroke="black" d="M219.75,-354.14C251.6,-343.58 297.28,-328.42 334.02,-316.23"/>
 71 | <polygon fill="black" stroke="black" points="335.25,-319.51 343.64,-313.04 333.05,-312.87 335.25,-319.51"/>
 72 | </g>
 73 | <!-- graphop&#45;&gt;a -->
 74 | <g id="edge9" class="edge">
 75 | <title>graphop&#45;&gt;a</title>
 76 | <path fill="none" stroke="#009999" stroke-width="3" stroke-dasharray="1,5" d="M172.5,-386.03C167.79,-396.42 162.41,-410.15 160,-423 154.78,-450.81 155.66,-483.32 157.29,-505.55"/>
 77 | <polygon fill="#009999" stroke="#009999" stroke-width="3" points="158.13,-515.58 152.81,-505.99 156.72,-510.68 156.3,-505.69 157.3,-505.61 158.29,-505.53 158.71,-510.51 161.78,-505.23 158.13,-515.58 158.13,-515.58"/>
 78 | <text text-anchor="middle" x="166.5" y="-436.6" font-family="bold" font-size="18.00" fill="#009999">1</text>
 79 | </g>
 80 | <!-- a_minus_ab -->
 81 | <g id="node6" class="node">
 82 | <title>a_minus_ab</title>
 83 | <polygon fill="none" stroke="#990000" stroke-width="3" points="223.5,-313 140.5,-313 140.5,-277 223.5,-277 223.5,-313"/>
 84 | <text text-anchor="middle" x="182" y="-291.3" font-family="Times,serif" font-size="14.00">a_minus_ab</text>
 85 | </g>
 86 | <!-- graphop&#45;&gt;a_minus_ab -->
 87 | <g id="edge3" class="edge">
 88 | <title>graphop&#45;&gt;a_minus_ab</title>
 89 | <path fill="none" stroke="black" d="M181.24,-349.81C181.36,-341.79 181.49,-332.05 181.62,-323.07"/>
 90 | <polygon fill="black" stroke="black" points="185.12,-323.08 181.76,-313.03 178.12,-322.98 185.12,-323.08"/>
 91 | </g>
 92 | <!-- b&#45;&gt;graphop -->
 93 | <g id="edge5" class="edge">
 94 | <title>b&#45;&gt;graphop</title>
 95 | <path fill="none" stroke="black" stroke-dasharray="5,2" d="M110.12,-428.91C122.04,-418.84 139.96,-403.69 154.8,-391.15"/>
 96 | <polygon fill="black" stroke="black" points="157.29,-393.62 162.67,-384.49 152.77,-388.28 157.29,-393.62"/>
 97 | </g>
 98 | <!-- sub2 -->
 99 | <g id="node7" class="node">
100 | <title>sub2</title>
101 | <ellipse fill="none" stroke="black" cx="125" cy="-130" rx="33.29" ry="18"/>
102 | <text text-anchor="middle" x="125" y="-126.3" font-family="italic" font-size="14.00">sub2</text>
103 | </g>
104 | <!-- b&#45;&gt;sub2 -->
105 | <g id="edge11" class="edge">
106 | <title>b&#45;&gt;sub2</title>
107 | <path fill="none" stroke="#009999" stroke-width="3" stroke-dasharray="1,5" d="M89.72,-426.28C82.98,-412.28 74,-389.73 74,-369 74,-369 74,-369 74,-202 74,-182.72 86.56,-165.08 99.22,-152.21"/>
108 | <polygon fill="#009999" stroke="#009999" stroke-width="3" points="106.59,-145.21 102.44,-155.36 103.65,-149.38 100.03,-152.83 99.34,-152.1 98.65,-151.38 102.27,-147.93 96.24,-148.84 106.59,-145.21 106.59,-145.21"/>
109 | <text text-anchor="middle" x="80.5" y="-290.6" font-family="bold" font-size="18.00" fill="#009999">3</text>
110 | </g>
111 | <!-- a_minus_ab&#45;&gt;sub2 -->
112 | <g id="edge6" class="edge">
113 | <title>a_minus_ab&#45;&gt;sub2</title>
114 | <path fill="none" stroke="black" d="M189.5,-276.73C197.89,-255 209.17,-216.69 200,-185 194.67,-166.58 177.96,-153.58 161.67,-144.97"/>
115 | <polygon fill="black" stroke="black" points="162.74,-141.6 152.21,-140.39 159.69,-147.9 162.74,-141.6"/>
116 | </g>
117 | <!-- c -->
118 | <g id="node8" class="node">
119 | <title>c</title>
120 | <polygon fill="none" stroke="#990000" stroke-width="3" points="93,-197.44 120,-185 147,-197.44 146.97,-217.56 93.03,-217.56 93,-197.44"/>
121 | <text text-anchor="middle" x="120" y="-199.3" font-family="Times,serif" font-size="14.00">c</text>
122 | </g>
123 | <!-- a_minus_ab&#45;&gt;c -->
124 | <g id="edge13" class="edge">
125 | <title>a_minus_ab&#45;&gt;c</title>
126 | <path fill="none" stroke="#009999" stroke-width="3" stroke-dasharray="1,5" d="M170.05,-276.65C160.16,-262.3 146.06,-241.83 135.35,-226.28"/>
127 | <polygon fill="#009999" stroke="#009999" stroke-width="3" points="129.48,-217.77 138.86,-223.45 133.14,-221.32 135.98,-225.43 135.16,-226 134.33,-226.57 131.5,-222.45 131.45,-228.56 129.48,-217.77 129.48,-217.77"/>
128 | <text text-anchor="middle" x="162.5" y="-244.6" font-family="bold" font-size="18.00" fill="#009999">5</text>
129 | </g>
130 | <!-- sub2&#45;&gt;a_minus_ab -->
131 | <g id="edge12" class="edge">
132 | <title>sub2&#45;&gt;a_minus_ab</title>
133 | <path fill="none" stroke="#009999" stroke-width="3" stroke-dasharray="1,5" d="M135.74,-147.54C142.21,-157.98 150.27,-171.93 156,-185 166.1,-208.05 167.18,-214.52 173,-239 175.1,-247.83 176.87,-257.58 178.27,-266.39"/>
134 | <polygon fill="#009999" stroke="#009999" stroke-width="3" points="179.79,-276.53 173.86,-267.31 178.06,-271.74 177.32,-266.79 178.31,-266.64 179.3,-266.5 180.04,-271.44 182.76,-265.98 179.79,-276.53 179.79,-276.53"/>
135 | <text text-anchor="middle" x="174.5" y="-198.6" font-family="bold" font-size="18.00" fill="#009999">4</text>
136 | </g>
137 | <!-- a_minus_ab_minus_c -->
138 | <g id="node9" class="node">
139 | <title>a_minus_ab_minus_c</title>
140 | <polygon fill="wheat" stroke="black" points="234.02,-62.56 125,-75 15.98,-62.56 16.08,-42.44 233.92,-42.44 234.02,-62.56"/>
141 | <text text-anchor="middle" x="125" y="-53.3" font-family="Times,serif" font-size="14.00">a_minus_ab_minus_c</text>
142 | </g>
143 | <!-- sub2&#45;&gt;a_minus_ab_minus_c -->
144 | <g id="edge7" class="edge">
145 | <title>sub2&#45;&gt;a_minus_ab_minus_c</title>
146 | <path fill="none" stroke="black" d="M125,-111.81C125,-103.79 125,-94.05 125,-85.07"/>
147 | <polygon fill="black" stroke="black" points="128.5,-85.03 125,-75.03 121.5,-85.03 128.5,-85.03"/>
148 | </g>
149 | <!-- c&#45;&gt;sub2 -->
150 | <g id="edge8" class="edge">
151 | <title>c&#45;&gt;sub2</title>
152 | <path fill="none" stroke="black" d="M121.16,-185.53C121.72,-177.52 122.42,-167.68 123.06,-158.57"/>
153 | <polygon fill="black" stroke="black" points="126.57,-158.59 123.78,-148.37 119.58,-158.1 126.57,-158.59"/>
154 | </g>
155 | </g>
156 | </svg>
157 | 


--------------------------------------------------------------------------------
/docs/source/index.rst:
--------------------------------------------------------------------------------
  1 | 
  2 | ========
  3 | Graphtik
  4 | ========
  5 | 
  6 | |release|, |today| |gh-version| |pypi-version| |python-ver|
  7 | |dev-status| |ci-status| |doc-status| |cover-status|
  8 | |codestyle| |proj-lic|
  9 | 
 10 | |gh-watch| |gh-star| |gh-fork| |gh-issues|
 11 | 
 12 | .. default-role:: term
 13 | .. epigraph::
 14 |    It's a DAG all the way down!
 15 | 
 16 |    |sample-plot|
 17 | 
 18 | Computation graphs for Python & Pandas
 19 | --------------------------------------
 20 | **Graphtik** is a library to compose, solve, execute & plot *graphs of python functions*
 21 | (a.k.a `pipeline`\s) that consume and populate named data
 22 | (a.k.a `dependencies <dependency>`), whose names may be
 23 | `nested <hierarchical data>` (such as, *pandas* dataframe columns),
 24 | based on whether values for those dependencies exist in the inputs or
 25 | have been calculated earlier.
 26 | 
 27 | .. math-overview-start
 28 | 
 29 | In mathematical terms, given:
 30 | 
 31 | - a partially populated `data tree`, and
 32 | - `a set <pipeline>` of `functions operating <operation>` (consuming/producing) on
 33 |   `branches <dependency>` of the data tree,
 34 | 
 35 | *graphtik* `collects <compile>` a `subset of functions in a graph <plan>`
 36 | that when `executed <execute>` consume & produce as `values as possible in the data-tree
 37 | <solution>`.
 38 | 
 39 | |usage-overview|
 40 | 
 41 | .. math-overview-end
 42 | 
 43 | - Its primary use case is building flexible algorithms for data science/machine learning projects.
 44 | - It should be extendable to implement the following:
 45 | 
 46 |   - an `IoC dependency resolver <https://en.wikipedia.org/wiki/Dependency_injection>`_
 47 |     (e.g. Java Spring, Google Guice);
 48 |   - an executor of interdependent tasks based on files (e.g. GNU Make);
 49 |   - a custom ETL engine;
 50 |   - a spreadsheet calculation engine.
 51 | 
 52 | Graph\ **tik** `sprang <https://docs.google.com/spreadsheets/d/1HPgtg2l6v3uDS81hLOcFOZxIBLCnHGrcFOh3pFRIDio/edit#gid=0>`_
 53 | from `Graphkit`_ (summer 2019, v1.2.2) to :gh:`experiment <22>` with Python 3.6+ features,
 54 | but has diverged significantly with enhancements ever since.
 55 | 
 56 | .. raw:: html
 57 | 
 58 |    <details>
 59 |    <summary><a>Table of Contents</a></summary>
 60 | 
 61 | .. toctree::
 62 |    :maxdepth: 4
 63 |    :numbered: 1
 64 | 
 65 |    operations
 66 |    pipelines
 67 |    plotting
 68 |    arch
 69 |    reference
 70 |    Changes <changes>
 71 |    genindex
 72 | 
 73 | .. raw:: html
 74 | 
 75 |    </details>
 76 | 
 77 | 
 78 | .. _features:
 79 | 
 80 | .. include:: ../../README.rst
 81 |    :start-after:  .. _features:
 82 |    :end-before:  Quick start
 83 | 
 84 | .. _quick-start:
 85 | 
 86 | Quick start
 87 | -----------
 88 | Here's how to install::
 89 | 
 90 |    pip install graphtik
 91 | 
 92 | OR with dependencies for plotting support (and you need to install `Graphviz`_ program
 93 | separately with your OS tools)::
 94 | 
 95 |    pip install graphtik[plot]
 96 | 
 97 | 
 98 | Let's build a *graphtik* computation `pipeline` that produces the following
 99 | x3 `outputs` out of x2 `inputs` (``α`` and ``β``):
100 | 
101 | .. math::
102 |    :label: sample-formula
103 | 
104 |    α \times β
105 | 
106 |    α - α \times β
107 | 
108 |    |α - α \times β| ^ 3
109 | 
110 | ..
111 | 
112 |    >>> from graphtik import compose, operation
113 |    >>> from operator import mul, sub
114 | 
115 |    >>> @operation(name="abs qubed",
116 |    ...            needs=["α-α×β"],
117 |    ...            provides=["|α-α×β|³"])
118 |    ... def abs_qubed(a):
119 |    ...    return abs(a) ** 3
120 | 
121 | .. hint::
122 |    Notice that *graphtik* has not problem working in unicode chars
123 |    for `dependency` names.
124 | 
125 | Compose the ``abspow`` function along with ``mul`` & ``sub``  built-ins
126 | into a computation `graph`:
127 | 
128 |    >>> graphop = compose("graphop",
129 |    ...    operation(mul, needs=["α", "β"], provides=["α×β"]),
130 |    ...    operation(sub, needs=["α", "α×β"], provides=["α-α×β"]),
131 |    ...    abs_qubed,
132 |    ... )
133 |    >>> graphop
134 |    Pipeline('graphop', needs=['α', 'β', 'α×β', 'α-α×β'],
135 |             provides=['α×β', 'α-α×β', '|α-α×β|³'],
136 |             x3 ops: mul, sub, abs qubed)
137 | 
138 | You may plot the function graph in a file like this
139 | (if in *jupyter*, no need to specify the file, see :ref:`jupyter_rendering`):
140 | 
141 |    >>> graphop.plot('graphop.svg')      # doctest: +SKIP
142 | 
143 | .. graphtik::
144 | 
145 | As you can see, any function can be used as an operation in Graphtik,
146 | even ones imported from system modules.
147 | 
148 | Run the graph-operation and request all of the outputs:
149 | 
150 |    >>> sol = graphop(**{'α': 2, 'β': 5})
151 |    >>> sol
152 |    {'α': 2, 'β': 5, 'α×β': 10, 'α-α×β': -8, '|α-α×β|³': 512}
153 | 
154 | `Solutions <solution>` are `plottable` as well:
155 | 
156 |    >>> solution.plot('solution.svg')      # doctest: +SKIP
157 | 
158 | .. graphtik::
159 | 
160 | Run the graph-operation and request a subset of the outputs:
161 | 
162 |    >>> solution = graphop.compute({'α': 2, 'β': 5}, outputs=["α-α×β"])
163 |    >>> solution
164 |    {'α-α×β': -8}
165 | 
166 | .. graphtik::
167 | 
168 | ... where the (interactive) legend is this:
169 | 
170 | .. graphtik::
171 |    :width: 65%
172 |    :name: legend
173 | 
174 |    >>> from graphtik.plot import legend
175 |    >>> l = legend()
176 | 
177 | .. default-role:: obj
178 | .. |sample-plot| raw:: html
179 |     :file:  images/sample.svg
180 | .. |usage-overview| image:: images/GraphkitUsageOverview.svg
181 |     :alt: Usage overview of graphtik library
182 |     :width: 640px
183 |     :align: middle
184 | .. from https://docs.google.com/document/d/1P73jgcAEzR_Vw491DQR0zogdunJOj3qh0h_lvphdaHk
185 | .. include:: ../../README.rst
186 |     :start-after: _badges_substs:


--------------------------------------------------------------------------------
/docs/source/operations.rst:
--------------------------------------------------------------------------------
  1 | Operations
  2 | ==========
  3 | 
  4 | An :term:`operation` is a function in a computation :term:`pipeline`,
  5 | abstractly represented by the :class:`.Operation` class.
  6 | This class specifies the :term:`dependencies <dependency>` forming the *pipeline*'s
  7 | :term:`network`.
  8 | 
  9 | 
 10 | Defining Operations
 11 | -------------------
 12 | You may inherit the :class:`.Operation` abstract class to do the following:
 13 | 
 14 | - define the :term:`needs` & :term:`provides` properties as collection of@ :term:`dependencies
 15 |   <dependency>` (needed to solve the dependencies :term:`network`),
 16 | - override the ``compute(solution)`` method  to read from the :term:`solution` argument
 17 |   those values listed in `needs` (those values only are guaranteed to exist when called),
 18 | - do some business, and then
 19 | - populate the values listed in `provides` back into `solution`
 20 |   (if other values are populated, they may be ignored).
 21 | 
 22 | 
 23 | But there is an easier way -- actually half of the code in this project is dedicated
 24 | to retrofitting existing *functions* unaware of all these, into :term:`operation`\s.
 25 | 
 26 | Operations from existing functions
 27 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 28 | The :class:`.FnOp` provides a concrete wrapper around any arbitrary function
 29 | to define and execute within a *pipeline*.
 30 | Use the :func:`.operation()` factory to instantiate one:
 31 | 
 32 |    >>> from operator import add
 33 |    >>> from graphtik import operation
 34 | 
 35 |    >>> add_op = operation(add,
 36 |    ...                    needs=['a', 'b'],
 37 |    ...                    provides=['a_plus_b'])
 38 |    >>> add_op
 39 |    FnOp(name='add', needs=['a', 'b'], provides=['a_plus_b'], fn='add')
 40 | 
 41 | You may still call the original function at :attr:`.FnOp.fn`,
 42 | bypassing thus any operation pre-processing:
 43 | 
 44 |       >>> add_op.fn(3, 4)
 45 |       7
 46 | 
 47 | But the proper way is to *call the operation* (either directly or by calling the
 48 | :meth:`.FnOp.compute()` method).  Notice though that unnamed
 49 | positional parameters are not supported:
 50 | 
 51 |       >>> add_op(a=3, b=4)
 52 |       {'a_plus_b': 7}
 53 | 
 54 | .. tip::
 55 |    (unstable API) In case your function needs to access the :mod:`.execution` machinery
 56 |    or its wrapping operation, it can do that through the :data:`.task_context`
 57 |    (unstable API, not working during (deprecated) :term:`parallel execution`,
 58 |    see :ref:`task-context`)
 59 | 
 60 | 
 61 | Builder pattern
 62 | ^^^^^^^^^^^^^^^
 63 | There are two ways to instantiate a :class:`.FnOp`\s, each one suitable
 64 | for different scenarios.
 65 | 
 66 | We've seen that calling manually :func:`.operation()` allows putting into a pipeline
 67 | functions that are defined elsewhere (e.g. in another module, or are system functions).
 68 | 
 69 | But that method is also useful if you want to create multiple operation instances
 70 | with similar attributes, e.g. ``needs``:
 71 | 
 72 |    >>> op_factory = operation(needs=['a'])
 73 | 
 74 | Notice that we specified a `fn`, in order to get back a :class:`.FnOp`
 75 | instance (and not a decorator).
 76 | 
 77 |    >>> from graphtik import operation, compose
 78 |    >>> from functools import partial
 79 | 
 80 |    >>> def mypow(a, p=2):
 81 |    ...    return a ** p
 82 | 
 83 |    >>> pow_op2 = op_factory.withset(fn=mypow, provides="^2")
 84 |    >>> pow_op3 = op_factory.withset(fn=partial(mypow, p=3), name='pow_3', provides='^3')
 85 |    >>> pow_op0 = op_factory.withset(fn=lambda a: 1, name='pow_0', provides='^0')
 86 | 
 87 |    >>> graphop = compose('powers', pow_op2, pow_op3, pow_op0)
 88 |    >>> graphop
 89 |    Pipeline('powers', needs=['a'], provides=['^2', '^3', '^0'], x3 ops:
 90 |       mypow, pow_3, pow_0)
 91 | 
 92 | 
 93 |    >>> graphop(a=2)
 94 |    {'a': 2, '^2': 4, '^3': 8, '^0': 1}
 95 | 
 96 | .. graphtik::
 97 | .. Tip::
 98 |   See :ref:`plotting` on how to make diagrams like this.
 99 | 
100 | 
101 | Decorator specification
102 | ^^^^^^^^^^^^^^^^^^^^^^^
103 | 
104 | If you are defining your computation graph and the functions that comprise it all in the same script,
105 | the decorator specification of ``operation`` instances might be particularly useful,
106 | as it allows you to assign computation graph structure to functions as they are defined.
107 | Here's an example:
108 | 
109 |    >>> from graphtik import operation, compose
110 | 
111 |    >>> @operation(needs=['b', 'a', 'r'], provides='bar')
112 |    ... def foo(a, b, c):
113 |    ...   return c * (a + b)
114 | 
115 |    >>> graphop = compose('foo_graph', foo)
116 | 
117 | .. graphtik::
118 | 
119 | - Notice that if ``name`` is not given, it is deduced from the function name.
120 | 
121 | 
122 | Specifying graph structure: ``provides`` and ``needs``
123 | ------------------------------------------------------
124 | Each :term:`operation` is a node in a computation :term:`graph`,
125 | depending and supplying data from and to other nodes (via the :term:`solution`),
126 | in order to :term:`compute`.
127 | 
128 | This graph structure is specified (mostly) via the ``provides`` and ``needs`` arguments
129 | to the :func:`.operation` factory, specifically:
130 | 
131 | ``needs``
132 |    this argument names the list of (positionally ordered) :term:`inputs` data the `operation`
133 |    requires to receive from *solution*.
134 |    The list corresponds, roughly, to the arguments of the underlying function
135 |    (plus any :term:`tokens`).
136 | 
137 |    It can be a single string, in which case a 1-element iterable is assumed.
138 | 
139 |    :seealso: :term:`needs`, :term:`modifier`, :attr:`.FnOp.needs`,
140 |       :attr:`.FnOp._user_needs`, :attr:`.FnOp._fn_needs`
141 | 
142 | ``provides``
143 |    this argument names the list of (positionally ordered) :term:`outputs` data
144 |    the operation provides into the *solution*.
145 |    The list corresponds, roughly, to the returned values of the `fn`
146 |    (plus any :term:`tokens` & :term:`alias`\es).
147 | 
148 |    It can be a single string, in which case a 1-element iterable is assumed.
149 | 
150 |    If they are more than one, the underlying function must return an iterable
151 |    with same number of elements (unless it :term:`returns dictionary`).
152 | 
153 |    :seealso: :term:`provides`, :term:`modifier`, :attr:`.FnOp.provides`,
154 |       :attr:`.FnOp._user_provides`, :attr:`.FnOp._fn_provides`
155 | 
156 | Declarations of *needs* and *provides* is affected by :term:`modifier`\s like
157 | :func:`.keyword`:
158 | 
159 | Map inputs(& outputs) to differently named function arguments (& results)
160 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
161 | .. autofunction:: graphtik.modifier.keyword
162 |    :noindex:
163 | 
164 | Operations may execute with missing inputs
165 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
166 | .. autofunction:: graphtik.modifier.optional
167 |    :noindex:
168 | 
169 | Calling functions with varargs (``*args``)
170 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
171 | .. autofunction:: graphtik.modifier.vararg
172 |    :noindex:
173 | .. autofunction:: graphtik.modifier.varargs
174 |    :noindex:
175 | 
176 | 
177 | .. _aliases:
178 | 
179 | Interface differently named dependencies: aliases & keyword modifier
180 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
181 | Sometimes, you need to interface functions & operations where they name a
182 | :term:`dependency` differently.  There are 4 different ways to accomplish that:
183 | 
184 | 
185 | 1. Introduce some :term:`"pipe-through" operation <conveyor operation>`
186 |    (see the example in :ref:`conveyor-function`, below).
187 | 
188 | 2. Annotate certain `needs` with :func:`.keyword` *modifier*
189 |    (exemplified in the modifier).
190 | 
191 | 3. For a :term:`returns dictionary` operation, annotate certain `provides`
192 |    with a :func:`.keyword` *modifier* (exemplified in the modifier).
193 | 
194 | 4. :term:`Alias <alias>` (clone) certain `provides` to different names:
195 | 
196 |        >>> op = operation(str,
197 |        ...                name="cloning `provides` with an `alias`",
198 |        ...                provides="real thing",
199 |        ...                aliases={"real thing": "clone"})
200 | 
201 |    .. graphtik::
202 | 
203 | .. _conveyor-function:
204 | 
205 | Default conveyor operation
206 | ^^^^^^^^^^^^^^^^^^^^^^^^^^
207 | If you don't specify a callable, the :term:`default identity function` get assigned,
208 | as long a `name` for the operation is given, and the number of `needs` matches
209 | the number of `provides`.
210 | 
211 | This facilitates conveying :term:`inputs` into renamed :term:`outputs` without the need
212 | to define a trivial *identity function* matching the `needs` & `provides` each time:
213 | 
214 |     >>> from graphtik import keyword, optional, vararg
215 |     >>> op = operation(
216 |     ...     None,
217 |     ...     name="a",
218 |     ...     needs=[optional("opt"), vararg("vararg"), "pos", keyword("kw")],
219 |     ...     # positional vararg, keyword, optional
220 |     ...     provides=["pos", "vararg", "kw", "opt"],
221 |     ... )
222 |     >>> op(opt=5, vararg=6, pos=7, kw=8)
223 |     {'pos': 7, 'vararg': 6, 'kw': 5, 'opt': 8}
224 | 
225 | Notice that the order of the results is not that of the `needs`
226 | (or that of the `inputs` in the ``compute()`` method), but, as explained in the comment-line,
227 | it follows Python semantics.
228 | 
229 | 
230 | Considerations for when building pipelines
231 | ------------------------------------------
232 | When many operations are composed into a computation graph, Graphtik matches up
233 | the values in their *needs* and *provides* to form the edges of that graph
234 | (see :ref:`graph-composition` for more on that), like the operations from
235 | the sample formula :eq:`sample-formula` in :ref:`quick-start` section:
236 | 
237 |    >>> from operator import mul, sub
238 |    >>> from functools import partial
239 |    >>> from graphtik import compose, operation
240 | 
241 |    >>> def abspow(a, p):
242 |    ...   """Compute |a|^p. """
243 |    ...   c = abs(a) ** p
244 |    ...   return c
245 | 
246 |    >>> # Compose the mul, sub, and abspow operations into a computation graph.
247 |    >>> graphop = compose("graphop",
248 |    ...    operation(mul, needs=["α", "β"], provides=["α×β"]),
249 |    ...    operation(sub, needs=["α", "α×β"], provides=["α-α×β"]),
250 |    ...    operation(name="abspow1", needs=["α-α×β"], provides=["|α-α×β|³"])
251 |    ...    (partial(abspow, p=3))
252 |    ... )
253 |    >>> graphop
254 |    Pipeline('graphop',
255 |                     needs=['α', 'β', 'α×β', 'α-α×β'],
256 |                     provides=['α×β', 'α-α×β', '|α-α×β|³'],
257 |                     x3 ops: mul, sub, abspow1)
258 | 
259 | 
260 | - Notice the use of :func:`functools.partial()` to set parameter ``p`` to a constant value.
261 | - And this is done by calling once more the returned "decorator" from :func:`operation()`,
262 |   when called without a function.
263 | 
264 | The ``needs`` and ``provides`` arguments to the operations in this script define
265 | a computation graph that looks like this:
266 | 
267 | .. graphtik::
268 | 


--------------------------------------------------------------------------------
/docs/source/reference.rst:
--------------------------------------------------------------------------------
  1 | =============
  2 | API Reference
  3 | =============
  4 | 
  5 | .. autosummary::
  6 | 
  7 |      graphtik
  8 |      graphtik.fnop
  9 |      graphtik.autograph
 10 |      graphtik.pipeline
 11 |      graphtik.modifier
 12 |      graphtik.planning
 13 |      graphtik.execution
 14 |      graphtik.plot
 15 |      graphtik.config
 16 |      graphtik.base
 17 |      graphtik.jetsam
 18 |      graphtik.jsonpointer
 19 |      graphtik.sphinxext
 20 | 
 21 | .. graphviz::
 22 |      GraphtikModuleDependencies.dot
 23 | 
 24 | Package: `graphtik`
 25 | ===================
 26 | .. automodule:: graphtik
 27 |      :members:
 28 |      :undoc-members:
 29 |      :private-members:
 30 |      :special-members:
 31 | 
 32 | Module: `fnop`
 33 | ==============
 34 | 
 35 | .. automodule:: graphtik.fnop
 36 |      :members:
 37 |      :undoc-members:
 38 |      :private-members:
 39 |      :special-members:
 40 | 
 41 | .. automodule:: graphtik.autograph
 42 |      :members:
 43 |      :undoc-members:
 44 |      :private-members:
 45 |      :special-members:
 46 | 
 47 | Module: `pipeline`
 48 | ==================
 49 | 
 50 | .. automodule:: graphtik.pipeline
 51 |      :members:
 52 |      :undoc-members:
 53 |      :private-members:
 54 |      :special-members:
 55 | 
 56 | Module: `modifier`
 57 | ==================
 58 | 
 59 | .. automodule:: graphtik.modifier
 60 |      :members:
 61 |      :private-members:
 62 |      :special-members:
 63 | 
 64 | Module: `planning`
 65 | ==================
 66 | 
 67 | .. automodule:: graphtik.planning
 68 |      :members:
 69 |      :private-members:
 70 |      :special-members:
 71 |      :undoc-members:
 72 | 
 73 | Module: `execution`
 74 | ===================
 75 | 
 76 | .. automodule:: graphtik.execution
 77 |      :members:
 78 |      :private-members:
 79 |      :special-members:
 80 |      :undoc-members:
 81 | 
 82 | Module: `plot`
 83 | ==============
 84 | 
 85 | .. automodule:: graphtik.plot
 86 |      :members:
 87 |      :undoc-members:
 88 | 
 89 | Module: `config`
 90 | ================
 91 | 
 92 | .. automodule:: graphtik.config
 93 |      :members:
 94 | 
 95 | Module: `base`
 96 | ==============
 97 | 
 98 | .. automodule:: graphtik.base
 99 |      :members:
100 |      :undoc-members:
101 | 
102 | Module: `jetsam`
103 | ================
104 | 
105 | .. automodule:: graphtik.jetsam
106 |      :members:
107 |      :undoc-members:
108 | 
109 | Module: `jsonpointer`
110 | =====================
111 | 
112 | .. automodule:: graphtik.jsonpointer
113 |      :members:
114 | 
115 | Package: `sphinxext`
116 | ====================
117 | 
118 | .. automodule:: graphtik.sphinxext
119 |      :members:
120 | 


--------------------------------------------------------------------------------
/graphtik/__init__.py:
--------------------------------------------------------------------------------
 1 | # Copyright 2016, Yahoo Inc.
 2 | # Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms.
 3 | """
 4 | :term:`computation` graphs for Python & Pandas
 5 | 
 6 | .. import-speeds-start
 7 | .. tip::
 8 |      The module *import-time* dependencies have been carefully optimized so that
 9 |      importing all from package takes the minimum time (e.g. *<10ms* in a 2019 laptop)::
10 | 
11 |         >>> %time from graphtik import *     # doctest: +SKIP
12 |         CPU times: user 8.32 ms, sys: 34 µs, total: 8.35 ms
13 |         Wall time: 7.53 ms
14 | 
15 |      Still, constructing your :term:`pipeline`\\s on import time would take
16 |      considerable more time (e.g. *~300ms* for the 1st pipeline).
17 |      So prefer to construct them in "factory" module functions
18 |      (remember to annotate them with :pep:`typing hints <484>` to denote their retun type).
19 | 
20 | .. import-speeds-stop
21 | 
22 | .. seealso::
23 |     :func:`.plot.active_plotter_plugged()`, :func:`.plot.set_active_plotter()` &
24 |     :func:`.plot.get_active_plotter()` configs, not imported, unless plot is needed.
25 | """
26 | 
27 | __version__ = "11.0.0.dev0"
28 | __release_date__ = "24 Apr 2023, 23:45"
29 | __title__ = "graphtik"
30 | __summary__ = __doc__.splitlines()[0]
31 | __license__ = "Apache-2.0"
32 | __uri__ = "https://github.com/pygraphkit/graphtik"
33 | __author__ = "hnguyen, ankostis"  # chronologically ordered
34 | 
35 | 
36 | from .autograph import Autograph, FnHarvester, autographed
37 | from .base import AbortedException, IncompleteExecutionError
38 | from .fnop import NO_RESULT, NO_RESULT_BUT_SFX, operation
39 | from .modifier import (
40 |     hcat,
41 |     implicit,
42 |     keyword,
43 |     modify,
44 |     optional,
45 |     sfxed,
46 |     sfxed_vararg,
47 |     sfxed_varargs,
48 |     token,
49 |     vararg,
50 |     varargs,
51 |     vcat,
52 | )
53 | from .pipeline import compose
54 | 


--------------------------------------------------------------------------------
/graphtik/config.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2016, Yahoo Inc.
  2 | # Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms.
  3 | """
  4 | :term:`configurations` for network execution, and utilities on them.
  5 | 
  6 | .. seealso:: methods :func:`.plot.active_plotter_plugged()`, :func:`.plot.set_active_plotter()`,
  7 |     :func:`.plot.get_active_plotter()`
  8 | 
  9 |     Plot configrations were not defined here, not to pollute import space early,
 10 |     until they are actually needed.
 11 | 
 12 | .. note::
 13 |     The contant-manager function ``XXX_plugged()`` or ``XXX_enabled()`` do NOT launch
 14 |     their code blocks using :meth:`contextvars.Context.run()` in a separate "context",
 15 |     so any changes to these or other context-vars will persist
 16 |     (unless they are also done within such context-managers)
 17 | """
 18 | import ctypes
 19 | import os
 20 | from contextlib import contextmanager
 21 | from contextvars import ContextVar
 22 | from functools import partial
 23 | from multiprocessing import Value
 24 | from typing import Optional
 25 | 
 26 | _debug_env_var = os.environ.get("GRAPHTIK_DEBUG")
 27 | _debug: ContextVar[Optional[bool]] = ContextVar(
 28 |     "debug",
 29 |     default=_debug_env_var and (_debug_env_var.lower() not in "0 false off no".split()),
 30 | )
 31 | _abort: ContextVar[Optional[bool]] = ContextVar(
 32 |     "abort", default=Value(ctypes.c_bool, lock=False)
 33 | )
 34 | _skip_evictions: ContextVar[Optional[bool]] = ContextVar("skip_evictions", default=None)
 35 | _layered_solution: ContextVar[Optional[bool]] = ContextVar(
 36 |     "layered_solution", default=None
 37 | )
 38 | _execution_pool: ContextVar[Optional["Pool"]] = ContextVar(
 39 |     "execution_pool", default=None
 40 | )
 41 | _parallel_tasks: ContextVar[Optional[bool]] = ContextVar("parallel_tasks", default=None)
 42 | _marshal_tasks: ContextVar[Optional[bool]] = ContextVar("marshal_tasks", default=None)
 43 | _endure_operations: ContextVar[Optional[bool]] = ContextVar(
 44 |     "endure_operations", default=None
 45 | )
 46 | _reschedule_operations: ContextVar[Optional[bool]] = ContextVar(
 47 |     "reschedule_operations", default=None
 48 | )
 49 | 
 50 | 
 51 | def _getter(context_var) -> Optional[bool]:
 52 |     return context_var.get()
 53 | 
 54 | 
 55 | @contextmanager
 56 | def _tristate_set(context_var, enabled):
 57 |     return context_var.set(enabled if enabled is None else bool(enabled))
 58 | 
 59 | 
 60 | @contextmanager
 61 | def _tristate_armed(context_var: ContextVar, enabled=True):
 62 |     """Assumes "enabled" if `enabled` flag is None."""
 63 |     resetter = context_var.set(enabled if enabled is None else bool(enabled))
 64 |     try:
 65 |         yield
 66 |     finally:
 67 |         context_var.reset(resetter)
 68 | 
 69 | 
 70 | debug_enabled = partial(_tristate_armed, _debug)
 71 | """
 72 | Like :func:`set_debug()` as a context-manager, resetting back to old value.
 73 | 
 74 | .. seealso:: disclaimer about context-managers at the top of this :mod:`.config` module.
 75 | """
 76 | is_debug = partial(_getter, _debug)
 77 | """
 78 | Return :func:`.set_debug` or `True` if :envvar:`GRAPHTIK_DEBUG` not one of ``0 false off no``.
 79 | 
 80 | Affected behavior when :ref:`debug` enabled:
 81 | 
 82 | .. debug-behavior-start
 83 | 
 84 | + on errors, plots the 1st errored solution/plan/pipeline/net (in that order)
 85 |   in an SVG file inside the temp-directory, and its path is logged in ERROR-level;
 86 | + :term:`jetsam` logs in ERROR (instead of in DEBUG) all annotations on all calls
 87 |   up the stack trace (logged from ``graphtik.jetsam.err`` logger);
 88 | + :meth:`FnOp.compute()` prints out full given-inputs (not just their keys);
 89 | + net objects print more details recursively, like fields (not just op-names) and
 90 |   prune-comments;
 91 | + plotted SVG diagrams include style-provenance as tooltips;
 92 | + Sphinx extension also saves the original DOT file next to each image
 93 |   (see :confval:`graphtik_save_dot_files`).
 94 | 
 95 | .. debug-behavior-end
 96 | 
 97 | .. Note::
 98 |     The default is controlled with :envvar:`GRAPHTIK_DEBUG` environment variable.
 99 | 
100 | Note that enabling this flag is different from enabling logging in DEBUG,
101 | since it affects all code (eg interactive printing in debugger session,
102 | exceptions, doctests), not just debug statements (also affected by this flag).
103 | 
104 | :return:
105 |     a "reset" token (see :meth:`.ContextVar.set`)
106 | """
107 | set_debug = partial(_tristate_set, _debug)
108 | """
109 | Enable/disable debug-mode.
110 | 
111 | :param enabled:
112 |     - ``None, False, string(0, false, off, no)``: Disabled
113 |     - anything else: Enable DEBUG
114 | 
115 | see :func:`is_debug()`
116 | """
117 | 
118 | 
119 | def abort_run():
120 |     """
121 |     Sets the :term:`abort run` global flag, to halt all currently or future executing plans.
122 | 
123 |     This global flag is reset when any :meth:`.Pipeline.compute()` is executed,
124 |     or manually, by calling :func:`.reset_abort()`.
125 |     """
126 |     _abort.get().value = True
127 | 
128 | 
129 | def reset_abort():
130 |     """Reset the :term:`abort run` global flag, to permit plan executions to proceed."""
131 |     _abort.get().value = False
132 | 
133 | 
134 | def is_abort():
135 |     """Return `True` if networks have been signaled to stop :term:`execution`."""
136 |     return _abort.get().value
137 | 
138 | 
139 | evictions_skipped = partial(_tristate_armed, _skip_evictions)
140 | """
141 | Like :func:`set_skip_evictions()` as a context-manager, resetting back to old value.
142 | 
143 | .. seealso:: disclaimer about context-managers at the top of this :mod:`.config` module.
144 | """
145 | is_skip_evictions = partial(_getter, _skip_evictions)
146 | """see :func:`set_skip_evictions()`"""
147 | set_skip_evictions = partial(_tristate_set, _skip_evictions)
148 | """
149 | When true, disable globally :term:`eviction`\\s, to keep all intermediate solution values, ...
150 | 
151 | regardless of asked outputs.
152 | 
153 | :return:
154 |     a "reset" token (see :meth:`.ContextVar.set`)
155 | """
156 | 
157 | 
158 | solution_layered = partial(_tristate_armed, _layered_solution)
159 | """
160 | Like :func:`set_layered_solution()` as a context-manager, resetting back to old value.
161 | 
162 | .. seealso:: disclaimer about context-managers at the top of this :mod:`.config` module.
163 | """
164 | is_layered_solution = partial(_getter, _layered_solution)
165 | """see :func:`set_layered_solution()`"""
166 | set_layered_solution = partial(_tristate_set, _layered_solution)
167 | """
168 | whether to store operation results into separate :term:`solution layer`
169 | 
170 | :param enable:
171 |     If false/true, it overrides any param given when executing a pipeline or a plan.
172 |     If None (default), results are layered only if there are NO :term:`jsonp` dependencies
173 |     in the network.
174 | 
175 | :return:
176 |     a "reset" token (see :meth:`.ContextVar.set`)
177 | """
178 | 
179 | 
180 | @contextmanager
181 | def execution_pool_plugged(pool: "Optional[Pool]"):
182 |     """
183 |     Like :func:`set_execution_pool()` as a context-manager, resetting back to old value.
184 | 
185 |     .. seealso:: disclaimer about context-managers at the top of this :mod:`.config` module.
186 |     """
187 |     resetter = _execution_pool.set(pool)
188 |     try:
189 |         yield
190 |     finally:
191 |         _execution_pool.reset(resetter)
192 | 
193 | 
194 | def set_execution_pool(pool: "Optional[Pool]"):
195 |     """
196 |     (deprecated) Set the process-pool for :term:`parallel` plan executions.
197 | 
198 |     You may have to :also func:`set_marshal_tasks()` to resolve
199 |     pickling issues.
200 |     """
201 |     return _execution_pool.set(pool)
202 | 
203 | 
204 | def get_execution_pool() -> "Optional[Pool]":
205 |     """(deprecated) Get the process-pool for :term:`parallel` plan executions."""
206 |     return _execution_pool.get()
207 | 
208 | 
209 | tasks_in_parallel = partial(_tristate_armed, _parallel_tasks)
210 | """
211 | (deprecated) Like :func:`set_parallel_tasks()` as a context-manager, resetting back to old value.
212 | 
213 | .. seealso:: disclaimer about context-managers at the top of this :mod:`.config` module.
214 | """
215 | is_parallel_tasks = partial(_getter, _parallel_tasks)
216 | """see :func:`set_parallel_tasks()`"""
217 | set_parallel_tasks = partial(_tristate_set, _parallel_tasks)
218 | """
219 | Enable/disable globally :term:`parallel` execution of operations.
220 | 
221 | :param enable:
222 |     - If ``None`` (default), respect the respective flag on each operation;
223 |     - If true/false, force it for all operations.
224 | 
225 | :return:
226 |     a "reset" token (see :meth:`.ContextVar.set`)
227 | """
228 | 
229 | 
230 | tasks_marshalled = partial(_tristate_armed, _marshal_tasks)
231 | """
232 | (deprecated) Like :func:`set_marshal_tasks()` as a context-manager, resetting back to old value.
233 | 
234 | .. seealso:: disclaimer about context-managers at the top of this :mod:`.config` module.
235 | """
236 | is_marshal_tasks = partial(_getter, _marshal_tasks)
237 | """(deprecated) see :func:`set_marshal_tasks()`"""
238 | set_marshal_tasks = partial(_tristate_set, _marshal_tasks)
239 | """
240 | (deprecated) Enable/disable globally :term:`marshalling` of :term:`parallel` operations, ...
241 | 
242 | inputs & outputs with :mod:`dill`,  which might help for pickling problems.
243 | 
244 | :param enable:
245 |     - If ``None`` (default), respect the respective flag on each operation;
246 |     - If true/false, force it for all operations.
247 | 
248 | :return:
249 |     a "reset" token (see :meth:`.ContextVar.set`)
250 | """
251 | 
252 | 
253 | operations_endured = partial(_tristate_armed, _endure_operations)
254 | """
255 | Like :func:`set_endure_operations()` as a context-manager, resetting back to old value.
256 | 
257 | .. seealso:: disclaimer about context-managers at the top of this :mod:`.config` module.
258 | """
259 | is_endure_operations = partial(_getter, _endure_operations)
260 | """see :func:`set_endure_operations()`"""
261 | set_endure_operations = partial(_tristate_set, _endure_operations)
262 | """
263 | Enable/disable globally :term:`endurance` to keep executing even if some operations fail.
264 | 
265 | :param enable:
266 |     - If ``None`` (default), respect the flag on each operation;
267 |     - If true/false, force it for all operations.
268 | 
269 | :return:
270 |     a "reset" token (see :meth:`.ContextVar.set`)
271 | 
272 | ."""
273 | 
274 | 
275 | operations_reschedullled = partial(_tristate_armed, _reschedule_operations)
276 | """
277 | Like :func:`set_reschedule_operations()` as a context-manager, resetting back to old value.
278 | 
279 | .. seealso:: disclaimer about context-managers at the top of this :mod:`.config` module.
280 | """
281 | is_reschedule_operations = partial(_getter, _reschedule_operations)
282 | """see :func:`set_reschedule_operations()`"""
283 | set_reschedule_operations = partial(_tristate_set, _reschedule_operations)
284 | """
285 | Enable/disable globally :term:`rescheduling` for operations returning only *partial outputs*.
286 | 
287 | :param enable:
288 |     - If ``None`` (default), respect the flag on each operation;
289 |     - If true/false, force it for all operations.
290 | 
291 | :return:
292 |     a "reset" token (see :meth:`.ContextVar.set`)
293 | 
294 | ."""
295 | 


--------------------------------------------------------------------------------
/graphtik/jetsam.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2019-2020, Kostis Anagnostopoulos;
  2 | # Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms.
  3 | """:term:`jetsam` utility for annotating exceptions from ``locals()`` like :pep:`678`
  4 | 
  5 | PY3.11 exception-notes.
  6 | 
  7 | .. doctest::
  8 |     :hide:
  9 | 
 10 |     .. Workaround sphinx-doc/sphinx#6590
 11 | 
 12 |     >>> from graphtik.jetsam import *
 13 |     >>> __name__ = "graphtik.jetsam"
 14 | """
 15 | import logging
 16 | import sys
 17 | from contextlib import contextmanager
 18 | from pathlib import Path
 19 | 
 20 | log = logging.getLogger(__name__)
 21 | 
 22 | 
 23 | class Jetsam(dict):
 24 |     """
 25 |     The :term:`jetsam` is a dict with items accessed also as attributes.
 26 | 
 27 |     From https://stackoverflow.com/a/14620633/548792
 28 |     """
 29 | 
 30 |     def __init__(self, *args, **kwargs):
 31 |         super(Jetsam, self).__init__(*args, **kwargs)
 32 |         self.__dict__ = self
 33 | 
 34 |     def log_n_plot(self, plot=None) -> Path:
 35 |         """
 36 |         Log collected items, and plot 1st :term:`plottable` in a temp-file, if :ref:`debug`.
 37 | 
 38 |         :param plot:
 39 |             override DEBUG-flag if given (true, plots, false not)
 40 | 
 41 |         :return:
 42 |             the name of temp-file, also ERROR-logged along with the rest jetsam
 43 |         """
 44 |         from tempfile import gettempdir
 45 |         from textwrap import indent
 46 | 
 47 |         from . import __title__
 48 |         from .config import is_debug
 49 |         from .plot import save_plot_file_by_sha1
 50 | 
 51 |         debug = is_debug() if plot is None else plot
 52 | 
 53 |         ## Plot broken
 54 |         #
 55 |         plot_fpath = None
 56 |         if debug and "plot_fpath" not in self:
 57 |             for p_type in "solution plan pipeline network".split():
 58 |                 plottable = self.get(p_type)
 59 |                 if plottable is not None:
 60 |                     try:
 61 |                         dir_prefix = Path(gettempdir(), __title__)
 62 |                         plot_fpath = save_plot_file_by_sha1(plottable, dir_prefix)
 63 |                         self["plot_fpath"] = plot_fpath
 64 |                         break
 65 |                     except Exception as ex:
 66 |                         log.warning(
 67 |                             "Suppressed error while plotting jetsam %s: %s(%s)",
 68 |                             plottable,
 69 |                             type(ex).__name__,
 70 |                             ex,
 71 |                             exc_info=True,
 72 |                         )
 73 | 
 74 |         ## Log collected jetsam
 75 |         #
 76 |         #  NOTE: log jetsam only HERE (pipeline), to avoid repetitive printouts.
 77 |         #
 78 |         if self:
 79 |             items = "".join(
 80 |                 f"  +--{f'{k}({v.solid})' if hasattr(v, 'solid') else k}:"
 81 |                 f"\n{indent(str(v), ' ' * 4)}\n"
 82 |                 for k, v in self.items()
 83 |                 if v is not None
 84 |             )
 85 |             logging.getLogger(f"{__name__}.err").error("Salvaged jetsam:\n%s", items)
 86 | 
 87 |         return plot_fpath
 88 | 
 89 | 
 90 | def save_jetsam(ex, locs, *salvage_vars: str, annotation="jetsam", **salvage_mappings):
 91 |     """
 92 |     Annotate exception with salvaged values from locals(), log, (if :ref:`debug`) plot.
 93 | 
 94 |     :param ex:
 95 |         the exception to annotate
 96 |     :param locs:
 97 |         ``locals()`` from the context-manager's block containing vars
 98 |         to be salvaged in case of exception
 99 | 
100 |         ATTENTION: wrapped function must finally call ``locals()``, because
101 |         *locals* dictionary only reflects local-var changes after call.
102 |     :param annotation:
103 |         the name of the attribute to attach on the exception
104 |     :param salvage_vars:
105 |         local variable names to save as is in the salvaged annotations dictionary.
106 |     :param salvage_mappings:
107 |         a mapping of destination-annotation-keys --> source-locals-keys;
108 |         if a `source` is callable, the value to salvage is retrieved
109 |         by calling ``value(locs)``.
110 |         They take precedence over`salvage_vars`.
111 | 
112 |     :return:
113 |         the :class:`Jetsam` annotation, also attached on the exception
114 |     :raises:
115 |         any exception raised by the wrapped function, annotated with values
116 |         assigned as attributes on this context-manager
117 | 
118 |     - Any attributes attached on this manager are attached as a new dict on
119 |       the raised exception as new  ``jetsam`` attribute with a dict as value.
120 |     - If the exception is already annotated, any new items are inserted,
121 |       but existing ones are preserved.
122 |     - If :ref:`debug` is enabled, plots the 1st found errored in order
123 |       solution/plan/pipeline/net, and log its path.
124 | 
125 |     **Example:**
126 | 
127 |     Call it with managed-block's ``locals()`` and tell which of them to salvage
128 |     in case of errors::
129 | 
130 | 
131 |         >>> try:
132 |         ...     a = 1
133 |         ...     b = 2
134 |         ...     raise Exception("trouble!")
135 |         ... except Exception as ex:
136 |         ...     save_jetsam(ex, locals(), "a", b="salvaged_b", c_var="c")
137 |         ...     raise
138 |         Traceback (most recent call last):
139 |         Exception: trouble!
140 | 
141 |     And then from a REPL::
142 | 
143 |         >>> import sys
144 |         >>> sys.exc_info()[1].jetsam                # doctest: +SKIP
145 |         {'a': 1, 'salvaged_b': 2, "c_var": None}
146 | 
147 |     .. Note::
148 | 
149 |         In order not to obfuscate the landing position of post-mortem debuggers
150 |         in the case of errors, use the ``try-finally`` with ``ok`` flag pattern:
151 | 
152 |         >>> ok = False
153 |         >>> try:
154 |         ...
155 |         ...     pass        # do risky stuff
156 |         ...
157 |         ...     ok = True   # last statement in the try-body.
158 |         ... except Exception as ex:
159 |         ...     if not ok:
160 |         ...         ex = sys.exc_info()[1]
161 |         ...         save_jetsam(...)
162 | 
163 |     ** Reason:**
164 | 
165 |     Graphs may become arbitrary deep.  Debugging such graphs is notoriously hard.
166 | 
167 |     The purpose is not to require a debugger-session to inspect the root-causes
168 |     (without precluding one).
169 |     """
170 |     ## Fail EARLY before yielding on bad use.
171 |     #
172 |     assert isinstance(ex, Exception), ("Bad `ex`, not an exception dict:", ex)
173 |     assert isinstance(locs, dict), ("Bad `locs`, not a dict:", locs)
174 |     assert all(isinstance(i, str) for i in salvage_vars), (
175 |         "Bad `salvage_vars`!",
176 |         salvage_vars,
177 |     )
178 |     assert salvage_vars or salvage_mappings, "No `salvage_mappings` given!"
179 |     assert all(isinstance(v, str) or callable(v) for v in salvage_mappings.values()), (
180 |         "Bad `salvage_mappings`:",
181 |         salvage_mappings,
182 |     )
183 | 
184 |     ## Merge vars-mapping to save.
185 |     for var in salvage_vars:
186 |         if var not in salvage_mappings:
187 |             salvage_mappings[var] = var
188 | 
189 |     try:
190 |         jetsam = getattr(ex, annotation, None)
191 |         if not isinstance(jetsam, Jetsam):
192 |             jetsam = Jetsam()
193 |             setattr(ex, annotation, jetsam)
194 | 
195 |         ## Salvage those asked
196 |         for dst_key, src in salvage_mappings.items():
197 |             try:
198 |                 if dst_key not in jetsam:
199 |                     salvaged_value = src(locs) if callable(src) else locs.get(src)
200 |                     jetsam[dst_key] = salvaged_value
201 |             except Exception as ex:
202 |                 log.warning(
203 |                     "Suppressed error while salvaging jetsam item (%r, %r): %s(%s)",
204 |                     dst_key,
205 |                     src,
206 |                     type(ex).__name__,
207 |                     ex,
208 |                     exc_info=True,
209 |                 )
210 | 
211 |         return jetsam
212 |     except Exception as ex2:
213 |         log.warning(
214 |             "Suppressed error while annotating exception: %r", ex2, exc_info=True
215 |         )
216 | 


--------------------------------------------------------------------------------
/graphtik/py.typed:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/pygraphkit/graphtik/1079c1f85a3c60bd1f1f190a093021384a445a7e/graphtik/py.typed


--------------------------------------------------------------------------------
/graphtik/sphinxext/_graphtikbuilder.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2020, Kostis Anagnostopoulos.
  2 | # Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms.
  3 | """A builder that Render graphtik plots from doctest-runner's globals."""
  4 | from collections import OrderedDict
  5 | from pathlib import Path
  6 | from typing import Union
  7 | 
  8 | import pydot
  9 | from boltons.iterutils import first
 10 | from docutils import nodes
 11 | from sphinx.application import Sphinx
 12 | from sphinx.ext import doctest as extdoctest
 13 | from sphinx.locale import _, __
 14 | from sphinx.util import logging
 15 | 
 16 | from ..base import Plottable
 17 | from ..config import is_debug
 18 | from . import doctestglobs, dynaimage, graphtik_node
 19 | 
 20 | PlottableType = Union[None, Plottable, pydot.Dot]
 21 | 
 22 | log = logging.getLogger(__name__)
 23 | 
 24 | _image_mimetypes = {
 25 |     "svg": "image/svg+xml",
 26 |     "svgz": "image/svg+xml",
 27 |     "pdf": "image/x+pdf",
 28 | }
 29 | 
 30 | 
 31 | class HistoricDict(OrderedDict):
 32 |     def __setitem__(self, k, v):
 33 |         super().__setitem__(k, v)
 34 |         self.move_to_end(k)
 35 | 
 36 | 
 37 | class GraphtikPlotsBuilder(doctestglobs.ExposeGlobalsDocTestBuilder):
 38 |     """Retrieve a :term:`plottable` from doctests globals and render them."""
 39 | 
 40 |     run_empty_code = True
 41 | 
 42 |     def _make_group_globals(self, group: extdoctest.TestGroup):
 43 |         return HistoricDict()
 44 | 
 45 |     def _warn_out(self, text: str) -> None:
 46 |         """
 47 |         Silence warnings since errors unrelated to building site, ...
 48 | 
 49 |         unless :rst:confval`graphtik_warning_is_error` is true (default false).
 50 |         """
 51 |         if self.config.graphtik_warning_is_error:
 52 |             super()._warn_out(text)
 53 |         else:
 54 |             log.info(f"WARN-like: {text}", nonl=True)
 55 |         self.outfile.write(text)
 56 | 
 57 |     def _globals_updated(self, code: extdoctest.TestCode, globs: dict):
 58 |         """Collect plottable from doctest-runner globals and render graphtik plot."""
 59 |         node: nodes.Node = code.node.parent
 60 | 
 61 |         if isinstance(node, graphtik_node):
 62 |             plottable = self._retrieve_graphvar_plottable(
 63 |                 globs, node["graphvar"], (code.filename, code.lineno)
 64 |             )
 65 |             if plottable:
 66 |                 dot: pydot.Dot
 67 |                 if isinstance(plottable, pydot.Dot):
 68 |                     dot = plottable
 69 |                 else:
 70 |                     dot = plottable.plot(**self.config.graphtik_plot_keywords)
 71 |                 img_format = node["img_format"]
 72 |                 rel_img_path = self._render_dot_image(img_format, dot, node)
 73 |                 dot_str = (
 74 |                     f"{plottable.debugstr()}"
 75 |                     if hasattr(plottable, "debugstr")  # a Solution
 76 |                     else plottable
 77 |                 )
 78 |                 self._upd_image_node(
 79 |                     node, rel_img_path, dot_str=str(dot_str), cmap_id=dot.get_name()
 80 |                 )
 81 | 
 82 |     def _is_plottable(self, value):
 83 |         return isinstance(value, (Plottable, pydot.Dot))
 84 | 
 85 |     def _retrieve_graphvar_plottable(
 86 |         self, globs: dict, graphvar, location
 87 |     ) -> PlottableType:
 88 |         plottable: PlottableType = None
 89 | 
 90 |         if graphvar is None:
 91 |             ## Pick last plottable from globals.
 92 |             #
 93 |             for i, var in enumerate(reversed(list(globs))):
 94 |                 value = globs[var]
 95 |                 if self._is_plottable(value):
 96 |                     log.debug(
 97 |                         __(
 98 |                             "picked plottable %r from doctest globals (num %s from the end0)"
 99 |                         ),
100 |                         var,
101 |                         i,
102 |                         location=location,
103 |                     )
104 |                     plottable = value
105 |                     break
106 |             else:
107 |                 log.error(
108 |                     __("could not find any plottable in doctest globals"),
109 |                     location=location,
110 |                 )
111 | 
112 |         else:
113 |             ## Pick named var from globals.
114 |             #
115 |             try:
116 |                 value = globs[graphvar]
117 |                 if not self._is_plottable(value):
118 |                     log.warning(
119 |                         __(
120 |                             "value of graphvar %r in doctest globals is not plottable but %r"
121 |                         ),
122 |                         graphvar,
123 |                         type(value).__name__,
124 |                         location=location,
125 |                     )
126 |                 else:
127 |                     plottable = value
128 |             except KeyError:
129 |                 log.warning(
130 |                     __("could not find graphvar %r in doctest globals"),
131 |                     graphvar,
132 |                     location=location,
133 |                 )
134 | 
135 |         return plottable
136 | 
137 |     def _fill_graph_url(self, dot: pydot.Dot, fname: str) -> None:
138 |         """Add a link on the graph to open in new tab, if Dot has no URL."""
139 |         if "URL" not in dot.get_attributes():
140 |             dot.set_URL(fname)
141 |             if "target" not in dot.get_attributes():
142 |                 dot.set_target("_blank")
143 | 
144 |     def _render_dot_image(
145 |         self, img_format, dot: pydot.Dot, node: graphtik_node
146 |     ) -> Path:
147 |         """
148 |         Ensure png(+usemap)|svg|svgz|pdf file exist, and return its path.
149 | 
150 |         :raises:
151 |             any exception from Graphviz program
152 |         """
153 |         fname = f"{node['filename']}.{img_format}"
154 |         ## So that when clicked, a new tab opens
155 |         self._fill_graph_url(dot, fname)
156 | 
157 |         abs_fpath = Path(self.outdir, self.imagedir, fname)
158 |         log.info(__("Rendering '%s'..."), abs_fpath)
159 | 
160 |         save_dot_files = self.config.graphtik_save_dot_files
161 |         if save_dot_files is None:
162 |             save_dot_files = is_debug()
163 | 
164 |         abs_fpath.parent.mkdir(parents=True, exist_ok=True)
165 | 
166 |         ## Save dot-file before rendering,
167 |         #  to still have it in case of errors.
168 |         #
169 |         if save_dot_files:
170 |             self.env.graphtik_image_purgatory.register_doc_fpath(
171 |                 self.env.docname, abs_fpath.with_suffix(".txt")
172 |             )
173 |             with open(abs_fpath.with_suffix(".txt"), "w") as f:
174 |                 f.write(str(dot))
175 | 
176 |         self.env.graphtik_image_purgatory.register_doc_fpath(
177 |             self.env.docname, abs_fpath
178 |         )
179 |         dot.write(abs_fpath, format=img_format)
180 |         if img_format == "png":
181 |             cmap = dot.create(format="cmapx", encoding="utf-8").decode("utf-8")
182 |             node.cmap = cmap
183 | 
184 |         ## XXX: used to work till active-builder attributes were transferred to self.
185 |         # rel_fpath = Path(self.imgpath, fname)
186 |         rel_fpath = Path(self.imagedir, fname)
187 | 
188 |         return rel_fpath
189 | 
190 |     def _upd_image_node(
191 |         self, node: graphtik_node, rel_img_path: Path, dot_str: str, cmap_id: str
192 |     ):
193 |         img_format: str = node["img_format"]
194 |         assert img_format, (img_format, node)
195 | 
196 |         image_node: dynaimage = first(node.findall(dynaimage))
197 |         if img_format == "png":
198 |             image_node.tag = "img"
199 |             image_node["src"] = str(rel_img_path)
200 |             image_node["usemap"] = f"#{cmap_id}"
201 |             # HACK: graphtik-node not given to html-visitor.
202 |             image_node.cmap = getattr(node, "cmap", "")
203 |         else:
204 |             image_node.tag = "object"
205 |             # TODO: make sphinx-SVGs zoomable.
206 |             image_node["data"] = str(rel_img_path)
207 |             image_node["type"] = _image_mimetypes[img_format]
208 | 
209 |         if "alt" not in image_node:
210 |             image_node["alt"] = dot_str
211 | 
212 | 
213 | def get_graphtik_builder(app: Sphinx) -> GraphtikPlotsBuilder:
214 |     """Initialize a singleton patched doctest-builder"""
215 |     builder = getattr(app, "graphtik_builder", None)
216 |     if builder is None:
217 |         builder = GraphtikPlotsBuilder(app, app.env)
218 |         builder.imgpath = app.builder.imgpath
219 |         builder.imagedir = app.builder.imagedir
220 |         builder.init()
221 |         app.graphtik_builder = builder
222 | 
223 |     return builder
224 | 


--------------------------------------------------------------------------------
/graphtik/sphinxext/doctestglobs.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2020-2020, Kostis Anagnostopoulos;
  2 | # Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms.
  3 | """Patched doctest builder to expose doctest-runner's globals."""
  4 | import doctest
  5 | from doctest import DocTest, DocTestParser, DocTestRunner
  6 | from typing import Dict, List, Mapping
  7 | 
  8 | import sphinx
  9 | from docutils import nodes
 10 | from sphinx.application import Sphinx
 11 | from sphinx.ext import doctest as extdoctest
 12 | from sphinx.locale import _, __
 13 | from sphinx.util import logging
 14 | 
 15 | log = logging.getLogger(__name__)
 16 | 
 17 | 
 18 | class ExposeGlobalsDocTestBuilder(extdoctest.DocTestBuilder):
 19 |     """Patched to expose *globals* from executed doctests (even empty ones)."""
 20 | 
 21 |     name = "graphtik_plots"
 22 |     epilog = None
 23 |     run_empty_code = False
 24 | 
 25 |     def get_filename_for_node(self, node: nodes.Node, docname: str) -> str:
 26 |         """
 27 |         Return filename (possibly with docstring) since :meth:`get_line_number` inaccurate.
 28 | 
 29 |         Incomplete solution introduced by sphinx-doc/sphinx#4584
 30 |         """
 31 |         return node.source
 32 | 
 33 |     @staticmethod
 34 |     def get_line_number(node: nodes.Node) -> int:
 35 |         """
 36 |         Workaround the admition that parent class can't tell lineno.
 37 | 
 38 |         PATCHED because the line number is given relative to the stripped docstring,
 39 |         not the document, so report original filename in :meth:`get_filename_for_node()`
 40 |         above.
 41 | 
 42 |         Incomplete solution introduced by sphinx-doc/sphinx#4584
 43 |         """
 44 |         # TODO: find the root cause of this off by one error.
 45 |         return None if node.line is None else node.line - 1
 46 | 
 47 |     def test_doc(self, docname: str, doctree: nodes.Node) -> None:
 48 |         """
 49 |         HACK: Method overridden to annotate all TestCode instances with their nodes,
 50 | 
 51 |         so as to store back on them the value of `:graphvar:` in the doctest-runner globals,
 52 |         after they have been executed.
 53 |         """
 54 |         groups: Dict[str, extdoctest.TestGroup] = {}
 55 |         add_to_all_groups = []
 56 |         TestRunner = extdoctest.SphinxDocTestRunner
 57 |         self.setup_runner = TestRunner(verbose=False, optionflags=self.opt)
 58 |         self.test_runner = TestRunner(verbose=False, optionflags=self.opt)
 59 |         self.cleanup_runner = TestRunner(verbose=False, optionflags=self.opt)
 60 | 
 61 |         self.test_runner._fakeout = self.setup_runner._fakeout  # type: ignore
 62 |         self.cleanup_runner._fakeout = self.setup_runner._fakeout  # type: ignore
 63 | 
 64 |         if self.config.doctest_test_doctest_blocks:
 65 | 
 66 |             def condition(node: nodes.Node) -> bool:
 67 |                 return (
 68 |                     isinstance(node, (nodes.literal_block, nodes.comment))
 69 |                     and "testnodetype" in node
 70 |                 ) or isinstance(node, nodes.doctest_block)
 71 | 
 72 |         else:
 73 | 
 74 |             def condition(node: nodes.Node) -> bool:
 75 |                 return (
 76 |                     isinstance(node, (nodes.literal_block, nodes.comment))
 77 |                     and "testnodetype" in node
 78 |                 )
 79 | 
 80 |         for node in doctree.findall(condition):  # type: Element
 81 |             if self.skipped(node):
 82 |                 continue
 83 | 
 84 |             source = node["test"] if "test" in node else node.astext()
 85 |             filename = self.get_filename_for_node(node, docname)
 86 |             line_number = self.get_line_number(node)
 87 |             if not source and not self.run_empty_code:
 88 |                 log.warning(
 89 |                     __("no code/output in %s block"),
 90 |                     node.get("testnodetype", "doctest"),
 91 |                     location=(filename, line_number),
 92 |                 )
 93 |             code = extdoctest.TestCode(
 94 |                 source,
 95 |                 type=node.get("testnodetype", "doctest"),
 96 |                 filename=filename,
 97 |                 lineno=line_number,
 98 |                 options=node.get("options"),
 99 |             )
100 |             # HACK: annotate the TestCode with the node
101 |             # to store back plottable from doctest-runner globals.
102 |             code.node = node
103 | 
104 |             node_groups = node.get("groups", ["default"])
105 |             if "*" in node_groups:
106 |                 add_to_all_groups.append(code)
107 |                 continue
108 |             for groupname in node_groups:
109 |                 if groupname not in groups:
110 |                     groups[groupname] = extdoctest.TestGroup(groupname)
111 |                 groups[groupname].add_code(code)
112 |         for code in add_to_all_groups:
113 |             for group in groups.values():
114 |                 group.add_code(code)
115 |         if self.config.doctest_global_setup:
116 |             code = extdoctest.TestCode(
117 |                 self.config.doctest_global_setup, "testsetup", filename=None, lineno=0
118 |             )
119 |             for group in groups.values():
120 |                 group.add_code(code, prepend=True)
121 |         if self.config.doctest_global_cleanup:
122 |             code = extdoctest.TestCode(
123 |                 self.config.doctest_global_cleanup,
124 |                 "testcleanup",
125 |                 filename=None,
126 |                 lineno=0,
127 |             )
128 |             for group in groups.values():
129 |                 group.add_code(code)
130 |         if not groups:
131 |             return
132 | 
133 |         self._out("\nDocument: %s\n----------%s\n" % (docname, "-" * len(docname)))
134 |         for group in groups.values():
135 |             self.test_group(group)
136 |         # Separately count results from setup code
137 |         res_f, res_t = self.setup_runner.summarize(self._out, verbose=False)
138 |         self.setup_failures += res_f
139 |         self.setup_tries += res_t
140 |         if self.test_runner.tries:
141 |             res_f, res_t = self.test_runner.summarize(self._out, verbose=True)
142 |             self.total_failures += res_f
143 |             self.total_tries += res_t
144 |         if self.cleanup_runner.tries:
145 |             res_f, res_t = self.cleanup_runner.summarize(self._out, verbose=True)
146 |             self.cleanup_failures += res_f
147 |             self.cleanup_tries += res_t
148 | 
149 |     def _make_group_globals(self, group: dict):
150 |         return {}
151 | 
152 |     def test_group(self, group: extdoctest.TestGroup) -> None:
153 |         ns: Mapping = self._make_group_globals(group)
154 | 
155 |         def run_setup_cleanup(runner, testcodes, what):
156 |             # type: (Any, List[TestCode], Any) -> bool
157 |             examples = []
158 |             for testcode in testcodes:
159 |                 example = doctest.Example(testcode.code, "", lineno=testcode.lineno)
160 |                 examples.append(example)
161 |             if not examples:
162 |                 return True
163 |             # simulate a doctest with the code
164 |             sim_doctest = doctest.DocTest(
165 |                 examples,
166 |                 {},
167 |                 "%s (%s code)" % (group.name, what),
168 |                 testcodes[0].filename,
169 |                 0,
170 |                 None,
171 |             )
172 |             sim_doctest.globs = ns
173 |             old_f = runner.failures
174 |             self.type = "exec"  # the snippet may contain multiple statements
175 |             runner.run(sim_doctest, out=self._warn_out, clear_globs=False)
176 |             if runner.failures > old_f:
177 |                 return False
178 |             return True
179 | 
180 |         # run the setup code
181 |         if not run_setup_cleanup(self.setup_runner, group.setup, "setup"):
182 |             # if setup failed, don't run the group
183 |             return
184 | 
185 |         # run the tests
186 |         for code in group.tests:
187 |             py_code = code[0]
188 |             if len(code) == 1:
189 |                 # ordinary doctests (code/output interleaved)
190 |                 try:
191 |                     test = extdoctest.parser.get_doctest(
192 |                         py_code.code,
193 |                         {},
194 |                         group.name,  # type: ignore
195 |                         py_code.filename,
196 |                         py_code.lineno,
197 |                     )
198 |                 except Exception as ex:
199 |                     log.warning(
200 |                         __("ignoring invalid doctest code: %r\n due to: %s"),
201 |                         py_code.code,
202 |                         ex,
203 |                         location=(py_code.filename, py_code.lineno),
204 |                     )
205 |                     continue
206 | 
207 |                 # HACK: allow collecting vars even if code empty..
208 |                 if not test.examples and not self.run_empty_code:
209 |                     continue
210 | 
211 |                 for example in test.examples:
212 |                     # apply directive's comparison options
213 |                     new_opt = py_code.options.copy()
214 |                     new_opt.update(example.options)
215 |                     example.options = new_opt
216 |                 self.type = "single"  # as for ordinary doctests
217 |             else:
218 |                 # testcode and output separate
219 |                 output = code[1] and code[1].code or ""
220 |                 options = code[1] and code[1].options or {}
221 |                 # disable <BLANKLINE> processing as it is not needed
222 |                 options[doctest.DONT_ACCEPT_BLANKLINE] = True
223 |                 # find out if we're testing an exception
224 |                 m = extdoctest.parser._EXCEPTION_RE.match(output)  # type: ignore
225 |                 if m:
226 |                     exc_msg = m.group("msg")
227 |                 else:
228 |                     exc_msg = None
229 |                 example = doctest.Example(
230 |                     py_code.code,
231 |                     output,
232 |                     exc_msg=exc_msg,
233 |                     lineno=py_code.lineno,
234 |                     options=options,
235 |                 )
236 |                 test = doctest.DocTest(
237 |                     [example], {}, group.name, py_code.filename, py_code.lineno, None
238 |                 )
239 |                 self.type = "exec"  # multiple statements again
240 |             # DocTest.__init__ copies the globs namespace, which we don't want
241 |             test.globs = ns
242 |             # also don't clear the globs namespace after running the doctest
243 |             self.test_runner.run(test, out=self._warn_out, clear_globs=False)
244 | 
245 |             ## HACK: collect plottable from doctest-runner globals.
246 |             self._globals_updated(py_code, ns)
247 | 
248 |         # run the cleanup
249 |         run_setup_cleanup(self.cleanup_runner, group.cleanup, "cleanup")
250 | 
251 |     def _globals_updated(self, code: extdoctest.TestCode, globs: dict):
252 |         """Called after each test-code has executed."""
253 |         pass
254 | 


--------------------------------------------------------------------------------
/graphtik/sphinxext/graphtik.css:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2020-2020, Kostis Anagnostopoulos;
 3 |  * Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms.
 4 |  */
 5 | 
 6 | /** Applied on IMG or OBJECT */
 7 | .graphtik-zoomable-svg {
 8 |     border-style: inset;
 9 |     max-width: 100%;
10 | }
11 | 


--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
 1 | [build-system]
 2 | requires = ["setuptools>=30.3.0"]
 3 | build-backend = "setuptools.build_meta"
 4 | 
 5 | [tool.black]
 6 | ## Also `.gitignore` is sourced.
 7 | exclude = '''
 8 |     /(
 9 |         \.eggs
10 |         |.*venv.*
11 |         |\.git
12 |         |\.hg
13 |         |\.mypy_cache
14 |         |\.pytype
15 |         |\.nox
16 |         |\.tox
17 |         |build
18 |         |_build
19 |         |buck-out
20 |         |dist
21 |         |.vscode/.+
22 |     )/ | (
23 |     \..*cache.*
24 |     )
25 | '''
26 | 
27 | [tool.isort]
28 | profile = "black"
29 | remove_redundant_aliases = true
30 | src_paths = ["graphtik", "test", "docs/source"]


--------------------------------------------------------------------------------
/requirements-rtd.txt:
--------------------------------------------------------------------------------
1 | -e .[sphinx]


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
  1 | #
  2 | # This file is autogenerated by pip-compile with Python 3.11
  3 | # by the following command:
  4 | #
  5 | #    pip-compile --extra=all --resolver=backtracking setup.py
  6 | #
  7 | alabaster==0.7.13
  8 |     # via sphinx
  9 | astroid==2.15.4
 10 |     # via pylint
 11 | babel==2.12.1
 12 |     # via sphinx
 13 | black==23.3.0
 14 |     # via graphtik (setup.py)
 15 | bleach==6.0.0
 16 |     # via readme-renderer
 17 | boltons==23.0.0
 18 |     # via graphtik (setup.py)
 19 | build==0.10.0
 20 |     # via
 21 |     #   graphtik (setup.py)
 22 |     #   pip-tools
 23 | certifi==2022.12.7
 24 |     # via requests
 25 | charset-normalizer==3.1.0
 26 |     # via requests
 27 | click==8.1.3
 28 |     # via
 29 |     #   black
 30 |     #   pip-tools
 31 | contourpy==1.0.7
 32 |     # via matplotlib
 33 | coverage[toml]==7.2.3
 34 |     # via pytest-cov
 35 | cycler==0.11.0
 36 |     # via matplotlib
 37 | dill==0.3.6
 38 |     # via
 39 |     #   graphtik (setup.py)
 40 |     #   pylint
 41 | docutils==0.19
 42 |     # via
 43 |     #   readme-renderer
 44 |     #   sphinx
 45 | fonttools==4.39.3
 46 |     # via matplotlib
 47 | html5lib==1.1
 48 |     # via graphtik (setup.py)
 49 | idna==3.4
 50 |     # via requests
 51 | imagesize==1.4.1
 52 |     # via sphinx
 53 | iniconfig==2.0.0
 54 |     # via pytest
 55 | isort==5.12.0
 56 |     # via pylint
 57 | jinja2==3.1.2
 58 |     # via
 59 |     #   graphtik (setup.py)
 60 |     #   sphinx
 61 | kiwisolver==1.4.4
 62 |     # via matplotlib
 63 | lazy-object-proxy==1.9.0
 64 |     # via astroid
 65 | markupsafe==2.1.2
 66 |     # via
 67 |     #   graphtik (setup.py)
 68 |     #   jinja2
 69 | matplotlib==3.7.1
 70 |     # via
 71 |     #   graphtik (setup.py)
 72 |     #   sphinxext-opengraph
 73 | mccabe==0.7.0
 74 |     # via pylint
 75 | mypy==1.2.0
 76 |     # via graphtik (setup.py)
 77 | mypy-extensions==1.0.0
 78 |     # via
 79 |     #   black
 80 |     #   mypy
 81 | networkx==3.1
 82 |     # via graphtik (setup.py)
 83 | numpy==1.24.3
 84 |     # via
 85 |     #   contourpy
 86 |     #   matplotlib
 87 |     #   pandas
 88 | packaging==23.1
 89 |     # via
 90 |     #   black
 91 |     #   build
 92 |     #   matplotlib
 93 |     #   pytest
 94 |     #   pytest-sugar
 95 |     #   sphinx
 96 | pandas==2.0.1
 97 |     # via graphtik (setup.py)
 98 | pathspec==0.11.1
 99 |     # via black
100 | pillow==9.5.0
101 |     # via matplotlib
102 | pip-tools==6.13.0
103 |     # via graphtik (setup.py)
104 | platformdirs==3.4.0
105 |     # via
106 |     #   black
107 |     #   pylint
108 | pluggy==1.0.0
109 |     # via pytest
110 | pydot==1.4.2
111 |     # via graphtik (setup.py)
112 | pyenchant==3.2.2
113 |     # via sphinxcontrib-spelling
114 | pygments==2.15.1
115 |     # via
116 |     #   readme-renderer
117 |     #   sphinx
118 | pylint==2.17.3
119 |     # via graphtik (setup.py)
120 | pyparsing==3.0.9
121 |     # via
122 |     #   matplotlib
123 |     #   pydot
124 | pyproject-hooks==1.0.0
125 |     # via build
126 | pytest==7.3.1
127 |     # via
128 |     #   graphtik (setup.py)
129 |     #   pytest-cov
130 |     #   pytest-sphinx
131 |     #   pytest-sugar
132 | pytest-cov==4.0.0
133 |     # via graphtik (setup.py)
134 | pytest-sphinx==0.5.0
135 |     # via graphtik (setup.py)
136 | pytest-sugar==0.9.7
137 |     # via graphtik (setup.py)
138 | python-dateutil==2.8.2
139 |     # via
140 |     #   matplotlib
141 |     #   pandas
142 | pytz==2023.3
143 |     # via pandas
144 | readme-renderer==37.3
145 |     # via graphtik (setup.py)
146 | requests==2.29.0
147 |     # via sphinx
148 | six==1.16.0
149 |     # via
150 |     #   bleach
151 |     #   html5lib
152 |     #   python-dateutil
153 | snowballstemmer==2.2.0
154 |     # via sphinx
155 | sphinx==6.2.1
156 |     # via
157 |     #   graphtik (setup.py)
158 |     #   sphinxcontrib-spelling
159 |     #   sphinxext-opengraph
160 | sphinxcontrib-applehelp==1.0.4
161 |     # via sphinx
162 | sphinxcontrib-devhelp==1.0.2
163 |     # via sphinx
164 | sphinxcontrib-htmlhelp==2.0.1
165 |     # via sphinx
166 | sphinxcontrib-jsmath==1.0.1
167 |     # via sphinx
168 | sphinxcontrib-qthelp==1.0.3
169 |     # via sphinx
170 | sphinxcontrib-serializinghtml==1.1.5
171 |     # via sphinx
172 | sphinxcontrib-spelling==8.0.0
173 |     # via graphtik (setup.py)
174 | sphinxext-opengraph==0.8.2
175 |     # via graphtik (setup.py)
176 | termcolor==2.3.0
177 |     # via pytest-sugar
178 | tomlkit==0.11.7
179 |     # via pylint
180 | typing-extensions==4.5.0
181 |     # via mypy
182 | tzdata==2023.3
183 |     # via pandas
184 | urllib3==1.26.15
185 |     # via requests
186 | webencodings==0.5.1
187 |     # via
188 |     #   bleach
189 |     #   html5lib
190 | wheel==0.40.0
191 |     # via pip-tools
192 | wrapt==1.15.0
193 |     # via astroid
194 | 
195 | # The following packages are considered to be unsafe in a requirements file:
196 | # pip
197 | # setuptools
198 | 


--------------------------------------------------------------------------------
/setup.cfg:
--------------------------------------------------------------------------------
 1 | ## Python's setup.cfg for tool defaults:
 2 | #
 3 | [metadata]
 4 | ## Include CHANGES in wheel
 5 | # `license_files` it affects only "wheels" (not "sdists"), and defaults to ~5 files,
 6 | # see https://wheel.readthedocs.io/en/stable/user_guide.html#including-license-files-in-the-generated-wheel-file
 7 | #license_files = CHANGES.rst LICENSE
 8 | 
 9 | [bdist_wheel]
10 | universal = 1
11 | 
12 | 
13 | [tool:pytest]
14 | # See http://doc.pytest.org/en/latest/mark.html#mark
15 | markers =
16 |     slow: slow-running tests, run them: with: -m slow OR -m 'slow or not slow'
17 |     parallel: (slow) run pipelines in (deprecated) PARALLEL (thread or process pools)
18 |     thread: run pipelines in parallel with THREAD-pool (masthalled or not)
19 |     proc: (slow) run pipelines in parallel with PROCESS-pool (masthalled or not)
20 |     marshal: (slow) run pipelines in PARALLEL marshling (thread or process pools)
21 | 
22 | addopts =
23 |     --doctest-modules
24 |     # Faciltate developer, rum'em all with -m 'slow or not slow'.
25 |     -m 'not slow'
26 |     --ignore-glob=t.*
27 |     --ignore-glob=*venv*
28 |     # See https://stackoverflow.com/questions/42919871/avoid-no-commands-on-setup-py-with-py-test
29 |     --ignore=setup.py
30 |     # Version-from-sources trick fails due to different cwd.
31 |     --ignore=docs/source/conf.py
32 |     --doctest-report ndiff
33 |     --doctest-continue-on-failure
34 |     # --doctest-ignore-import-errors
35 |     --doctest-glob=*.md
36 |     --doctest-glob=*.rst
37 |     --cov-fail-under=80
38 | doctest_optionflags = NORMALIZE_WHITESPACE ELLIPSIS
39 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # Copyright 2016, Yahoo Inc.
  3 | # Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms.
  4 | import datetime as dt
  5 | import io
  6 | import os
  7 | import re
  8 | import subprocess as sbp
  9 | from typing import Optional
 10 | 
 11 | from setuptools import find_packages, setup
 12 | 
 13 | 
 14 | def _version() -> str:
 15 |     """
 16 |     Grab the version from the root package.
 17 |     """
 18 |     with io.open("graphtik/__init__.py", "rt", encoding="utf8") as f:
 19 |         return re.search(r'__version__ = "(.*?)"', f.read()).group(1).strip()
 20 | 
 21 | 
 22 | def _ask_git_version() -> Optional[str]:
 23 |     try:
 24 |         return sbp.check_output(
 25 |             "git describe --always".split(), universal_newlines=True
 26 |         ).strip()
 27 |     except Exception:
 28 |         pass
 29 | 
 30 | 
 31 | version = _version()
 32 | git_ver = _ask_git_version()
 33 | txt_ver = f"{version}+{git_ver}" if git_ver and git_ver != version else version
 34 | 
 35 | with open("README.rst") as f:
 36 |     long_description = (
 37 |         f.read()
 38 |         .replace("|version|", txt_ver)
 39 |         .replace("|release|", txt_ver)
 40 |         .replace("|today|", dt.datetime.now().isoformat())
 41 |     )
 42 |     long_description = re.sub(
 43 |         r":(?:ref|class|rst:dir):`([^`]+?)(?: <[^>]+>)?`", r"*\1*", long_description
 44 |     )
 45 | 
 46 | 
 47 | plot_deps = [
 48 |     "pydot",
 49 |     # filter/context decorators renamed, Markup()/escape() from MarkupSafe.
 50 |     "jinja2>=3",
 51 |     # Direct use now due to jinja2-3+.
 52 |     "MarkupSafe",
 53 | ]
 54 | matplot_deps = plot_deps + ["matplotlib"]
 55 | sphinx_deps = plot_deps + ["sphinx >=2", "sphinxext-opengraph"]
 56 | test_deps = list(
 57 |     set(
 58 |         matplot_deps
 59 |         + sphinx_deps
 60 |         + [
 61 |             "pytest",
 62 |             "pytest-sugar",
 63 |             "pytest-sphinx>=0.2.1",  # internal API changes
 64 |             "pytest-cov",
 65 |             "dill",
 66 |             "sphinxcontrib-spelling",
 67 |             "html5lib",  # for sphinxext TCs
 68 |             "readme-renderer",  # for PyPi landing-page
 69 |             "pandas",
 70 |         ]
 71 |     )
 72 | )
 73 | dev_deps = test_deps + ["build", "black", "pylint", "mypy", "pip-tools"]
 74 | 
 75 | setup(
 76 |     name="graphtik",
 77 |     version=version,
 78 |     description="A Python lib for solving & executing graphs of functions, with `pandas` in mind",
 79 |     long_description=long_description,
 80 |     long_description_content_type="text/x-rst",
 81 |     author="Kostis Anagnostopoulos, Huy Nguyen, Arel Cordero, Pierre Garrigues, Rob Hess, "
 82 |     "Tobi Baumgartner, Clayton Mellina",
 83 |     author_email="ankostis@gmail.com",
 84 |     url="http://github.com/pygraphkit/graphtik",
 85 |     project_urls={
 86 |         "Documentation": "https://graphtik.readthedocs.io/",
 87 |         "Release Notes": "https://graphtik.readthedocs.io/en/latest/changes.html",
 88 |         "Sources": "https://github.com/pygraphkit/graphtik",
 89 |         "Bug Tracker": "https://github.com/pygraphkit/graphtik/issues",
 90 |     },
 91 |     packages=find_packages(exclude=["test"]),
 92 |     package_data={
 93 |         "graphtik": ["py.typed"],
 94 |         "graphtik.sphinxext": ["*.css"],
 95 |     },
 96 |     python_requires=">=3.7",
 97 |     install_requires=[
 98 |         "networkx",
 99 |         "boltons",  # for IndexSet
100 |     ],
101 |     extras_require={
102 |         ## NOTE: update also "extras" in README/quick-start section .
103 |         "plot": plot_deps,
104 |         "matplot": matplot_deps,
105 |         "sphinx": sphinx_deps,
106 |         "test": test_deps,
107 |         # May help for pickling (deprecated) `parallel` tasks.
108 |         # See :term:`marshalling` and :func:`set_marshal_tasks()` configuration.
109 |         "dill": ["dill"],
110 |         "all": dev_deps,
111 |         "dev": dev_deps,
112 |     },
113 |     tests_require=test_deps,
114 |     license="Apache-2.0",
115 |     keywords=[
116 |         "graph",
117 |         "computation graph",
118 |         "DAG",
119 |         "directed acyclic graph",
120 |         "executor",
121 |         "scheduler",
122 |         "etl",
123 |         "workflow",
124 |         "pipeline",
125 |     ],
126 |     classifiers=[
127 |         "Development Status :: 4 - Beta",
128 |         "License :: OSI Approved :: Apache Software License",
129 |         "Intended Audience :: Developers",
130 |         "Intended Audience :: Science/Research",
131 |         "Natural Language :: English",
132 |         "Operating System :: MacOS :: MacOS X",
133 |         "Operating System :: Microsoft :: Windows",
134 |         "Operating System :: OS Independent",
135 |         "Operating System :: POSIX",
136 |         "Operating System :: Unix",
137 |         "Programming Language :: Python",
138 |         "Programming Language :: Python :: 3.7",
139 |         "Programming Language :: Python :: 3.8",
140 |         "Programming Language :: Python :: 3.9",
141 |         "Programming Language :: Python :: 3.10",
142 |         "Programming Language :: Python :: 3.11",
143 |         "Topic :: Software Development :: Libraries",
144 |         "Topic :: Software Development :: Libraries :: Python Modules",
145 |         "Topic :: Scientific/Engineering",
146 |         "Topic :: Software Development",
147 |     ],
148 |     zip_safe=True,
149 |     platforms="Windows,Linux,Solaris,Mac OS-X,Unix",
150 | )
151 | 


--------------------------------------------------------------------------------
/test/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2016, Yahoo Inc.
2 | # Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms.
3 | 


--------------------------------------------------------------------------------
/test/conftest.py:
--------------------------------------------------------------------------------
  1 | import logging
  2 | import os
  3 | from collections import namedtuple
  4 | from multiprocessing import Pool
  5 | from multiprocessing import dummy as mp_dummy
  6 | from multiprocessing import get_context
  7 | from operator import add
  8 | 
  9 | import pytest
 10 | 
 11 | from graphtik import compose, operation
 12 | from graphtik.config import debug_enabled, execution_pool_plugged, tasks_marshalled
 13 | 
 14 | from .helpers import (
 15 |     _marshal,
 16 |     _parallel,
 17 |     _proc,
 18 |     _slow,
 19 |     _thread,
 20 |     dilled,
 21 |     exe_params,
 22 |     pickled,
 23 | )
 24 | 
 25 | # Enable pytest-sphinx fixtures
 26 | # See https://www.sphinx-doc.org/en/master/devguide.html#unit-testing
 27 | pytest_plugins = "sphinx.testing.fixtures"
 28 | 
 29 | # TODO: is this needed along with norecursedirs?
 30 | # See https://stackoverflow.com/questions/33508060/create-and-import-helper-functions-in-tests-without-creating-packages-in-test-di
 31 | collect_ignore = ["helpers.py"]
 32 | 
 33 | 
 34 | ########
 35 | ## From https://stackoverflow.com/a/57002853/548792
 36 | ##
 37 | def pytest_addoption(parser):
 38 |     """Add a command line option to disable logger."""
 39 |     parser.addoption(
 40 |         "--logger-disabled",
 41 |         action="append",
 42 |         default=[],
 43 |         help="disable specific loggers",
 44 |     )
 45 | 
 46 | 
 47 | def pytest_configure(config):
 48 |     """Disable the loggers from CLI and silence sphinx markers warns."""
 49 |     for name in config.getoption("--logger-disabled", default=[]):
 50 |         logger = logging.getLogger(name)
 51 |         logger.propagate = False
 52 | 
 53 |     config.addinivalue_line("markers", "sphinx: parametrized sphinx test-launches")
 54 |     config.addinivalue_line(
 55 |         "markers", "test_params: for parametrized sphinx test-launches"
 56 |     )
 57 | 
 58 | 
 59 | ##
 60 | ########
 61 | 
 62 | 
 63 | @pytest.fixture
 64 | def debug_mode():
 65 |     from graphtik.config import debug_enabled
 66 | 
 67 |     with debug_enabled(True):
 68 |         yield
 69 | 
 70 | 
 71 | @pytest.fixture(params=[dilled, pickled])
 72 | def ser_method(request):
 73 |     return request.param
 74 | 
 75 | 
 76 | @pytest.fixture(
 77 |     params=[
 78 |         # PARALLEL?, Thread/Proc?, Marshalled?
 79 |         (None, None, None),
 80 |         pytest.param((1, 0, 0), marks=(_parallel, _thread)),
 81 |         pytest.param((1, 0, 1), marks=(_parallel, _thread, _marshal)),
 82 |         pytest.param((1, 1, 1), marks=(_parallel, _proc, _marshal, _slow)),
 83 |         pytest.param(
 84 |             (1, 1, 0),
 85 |             marks=(
 86 |                 _parallel,
 87 |                 _proc,
 88 |                 _slow,
 89 |                 pytest.mark.xfail(reason="ProcessPool non-marshaled may fail."),
 90 |             ),
 91 |         ),
 92 |     ]
 93 | )
 94 | def exemethod(request):
 95 |     """Returns exe-method combinations, and store them globally, for xfail checks."""
 96 |     parallel, proc_pool, marshal = request.param
 97 |     exe_params.parallel, exe_params.proc, exe_params.marshal = request.param
 98 | 
 99 |     nsharks = None  # number of pool swimmers....
100 | 
101 |     with tasks_marshalled(marshal):
102 |         if parallel:
103 |             if proc_pool:
104 |                 if os.name == "posix":  # Allthough it is the default ...
105 |                     # NOTE: "spawn" DEADLOCKS!.
106 |                     pool = get_context("fork").Pool(nsharks)
107 |                 else:
108 |                     pool = Pool(nsharks)
109 |             else:
110 |                 pool = mp_dummy.Pool(nsharks)
111 | 
112 |             with execution_pool_plugged(pool), pool:
113 |                 yield parallel
114 |         else:
115 |             yield parallel
116 | 
117 | 
118 | @pytest.fixture
119 | def samplenet():
120 |     """sum1 = (a + b), sum2 = (c + d), sum3 = c + (c + d)"""
121 |     sum_op1 = operation(name="sum_op1", needs=["a", "b"], provides="sum1")(add)
122 |     sum_op2 = operation(name="sum_op2", needs=["c", "d"], provides="sum2")(add)
123 |     sum_op3 = operation(name="sum_op3", needs=["c", "sum2"], provides="sum3")(add)
124 |     return compose("test_net", sum_op1, sum_op2, sum_op3)
125 | 
126 | 
127 | @pytest.fixture(params=[10, 20])
128 | def log_levels(request, caplog):
129 |     caplog.set_level(request.param)
130 |     return
131 | 


--------------------------------------------------------------------------------
/test/helpers.py:
--------------------------------------------------------------------------------
  1 | import pickle
  2 | import re
  3 | from collections import namedtuple
  4 | from dataclasses import dataclass
  5 | from itertools import chain, cycle
  6 | from pathlib import Path
  7 | from typing import Any, Union
  8 | from unittest.mock import MagicMock
  9 | 
 10 | import dill
 11 | import pytest
 12 | 
 13 | ###################################################
 14 | # Copied from <sphinx.git>/tests/test_build_html.py
 15 | 
 16 | _slow = pytest.mark.slow
 17 | _proc = pytest.mark.proc
 18 | _thread = pytest.mark.thread
 19 | _parallel = pytest.mark.parallel
 20 | _marshal = pytest.mark.marshal
 21 | 
 22 | 
 23 | @dataclass
 24 | class ExeParams:
 25 |     parallel: Any = "-"
 26 |     proc: Any = "-"
 27 |     marshal: Any = "-"
 28 | 
 29 | 
 30 | exe_params = ExeParams()
 31 | 
 32 | 
 33 | def oneliner(s) -> str:
 34 |     """Collapse any whitespace in stringified `s` into a single space."""
 35 |     return re.sub(r"[\n ]+", " ", str(s).strip())
 36 | 
 37 | 
 38 | def flat_dict(d):
 39 |     return chain.from_iterable(
 40 |         [zip(cycle([fname]), values) for fname, values in d.items()]
 41 |     )
 42 | 
 43 | 
 44 | def attr_check(attr, *regex, count: Union[int, None, bool] = None):
 45 |     """
 46 |     Asserts captured nodes have `attr` satisfying `regex` one by one (in a cycle).
 47 | 
 48 |     :param count:
 49 |         The number-of-nodes expected.
 50 |         If ``True``, this number becomes the number-of-`regex`;
 51 |         if none, no count check happens.
 52 |     """
 53 | 
 54 |     rexes = [re.compile(r) for r in regex]
 55 | 
 56 |     def checker(nodes):
 57 |         nonlocal count
 58 | 
 59 |         if count is not None:
 60 |             if count is True:
 61 |                 count = len(rexes)
 62 |             n = len(nodes)
 63 |             assert len(nodes) == count, f"expected {count} but found {n} nodes: {nodes}"
 64 | 
 65 |         for i, (node, rex, rex_pat) in enumerate(
 66 |             zip(nodes, cycle(rexes), cycle(regex))
 67 |         ):
 68 |             txt = node.get(attr)
 69 |             assert rex.search(
 70 |                 txt
 71 |             ), f"no0({i}) regex({rex_pat}) missmatched {node.tag}@%{attr}: {txt}"
 72 | 
 73 |     return checker
 74 | 
 75 | 
 76 | def check_xpath(etree, fname, path, check, be_found=True):
 77 |     nodes = list(etree.findall(path))
 78 |     if check is None:
 79 |         assert nodes == [], "found any nodes matching xpath " "%r in file %s" % (
 80 |             path,
 81 |             fname,
 82 |         )
 83 |         return
 84 |     else:
 85 |         assert nodes != [], "did not find any node matching xpath " "%r in file %s" % (
 86 |             path,
 87 |             fname,
 88 |         )
 89 |     if callable(check):
 90 |         check(nodes)
 91 |     elif not check:
 92 |         # only check for node presence
 93 |         pass
 94 |     else:
 95 | 
 96 |         def get_text(node):
 97 |             if node.text is not None:
 98 |                 # the node has only one text
 99 |                 return node.text
100 |             else:
101 |                 # the node has tags and text; gather texts just under the node
102 |                 return "".join(n.tail or "" for n in node)
103 | 
104 |         rex = re.compile(check)
105 |         if be_found:
106 |             if any(rex.search(get_text(node)) for node in nodes):
107 |                 return
108 |         else:
109 |             if all(not rex.search(get_text(node)) for node in nodes):
110 |                 return
111 | 
112 |         msg = "didn't match any" if be_found else "matched at least one"
113 |         assert False, (
114 |             f"{Path(fname).absolute()}:\n  {check!r} {msg} node at path {path!r}: "
115 |             f"{[node.text for node in nodes]}"
116 |         )
117 | 
118 | 
119 | def dilled(i):
120 |     return dill.loads(dill.dumps(i))
121 | 
122 | 
123 | def pickled(i):
124 |     return pickle.loads(pickle.dumps(i))
125 | 
126 | 
127 | def addall(*a, **kw):
128 |     "Same as a + b + ...."
129 |     return sum(a) + sum(kw.values())
130 | 
131 | 
132 | def abspow(a, p):
133 |     c = abs(a) ** p
134 |     return c
135 | 
136 | 
137 | def dummy_sol(named_inputs):
138 |     from graphtik.execution import Solution
139 | 
140 |     plan = MagicMock()
141 |     return Solution(plan, named_inputs)
142 | 


--------------------------------------------------------------------------------
/test/sphinxext/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/pygraphkit/graphtik/1079c1f85a3c60bd1f1f190a093021384a445a7e/test/sphinxext/__init__.py


--------------------------------------------------------------------------------
/test/sphinxext/conftest.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import shutil
 3 | 
 4 | import pytest
 5 | from sphinx.testing.path import path
 6 | 
 7 | # Exclude 'roots' dirs for pytest test collector
 8 | collect_ignore = ["roots"]
 9 | 
10 | 
11 | @pytest.fixture(scope="session")
12 | def rootdir():
13 |     return path(__file__).parent.abspath() / "roots"
14 | 
15 | 
16 | def _initialize_test_directory(session):
17 |     if "SPHINX_TEST_TEMPDIR" in os.environ:
18 |         tempdir = os.path.abspath(os.getenv("SPHINX_TEST_TEMPDIR"))
19 |         print("Temporary files will be placed in %s." % tempdir)
20 | 
21 |         if os.path.exists(tempdir):
22 |             shutil.rmtree(tempdir)
23 | 
24 |         os.makedirs(tempdir)
25 | 
26 | 
27 | def pytest_sessionstart(session):
28 |     _initialize_test_directory(session)
29 | 


--------------------------------------------------------------------------------
/test/sphinxext/roots/test-graphtik-directive/conf.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import sys
 3 | 
 4 | sys.path.insert(0, os.path.abspath("../../"))
 5 | 
 6 | extensions = [
 7 |     "graphtik.sphinxext",
 8 | ]
 9 | 
10 | latex_documents = [
11 |     (
12 |         "index",
13 |         "test.tex",
14 |         "The basic Sphinx documentation for testing",
15 |         "Sphinx",
16 |         "report",
17 |     )
18 | ]
19 | 
20 | # FIXME: htm5 dosn't work!??
21 | html_experimental_html5_writer = True
22 | 


--------------------------------------------------------------------------------
/test/sphinxext/roots/test-graphtik-directive/index.rst:
--------------------------------------------------------------------------------
  1 | 0. `graphtik` with :graphvar:
  2 | =============================
  3 | .. graphtik::
  4 |     :graphvar: pipeline1
  5 | 
  6 |     >>> from graphtik import compose, operation
  7 | 
  8 |     >>> pipeline1 = compose(
  9 |     ...     "pipeline1",
 10 |     ...     operation(name="op1", needs=["a", "b"], provides="aa")(lambda a, b: [a, b]),
 11 |     ... )
 12 | 
 13 | 
 14 | 1. Solved `graphtik` WITHOUT :graphvar:
 15 | =======================================
 16 | .. graphtik::
 17 |     :caption: Solved *pipeline2* with ``a=1``, ``b=2``
 18 | 
 19 |     >>> pipeline2 = compose(
 20 |     ...     "pipeline2",
 21 |     ...     operation(name="op1", needs=["a", "b"], provides="aa")(lambda a, b: [a, b]),
 22 |     ... )
 23 |     >>> sol = pipeline2(a=1, b=2)
 24 | 
 25 | 
 26 | 2. `graphtik` inherit from literal-block WITHOUT :graphvar:
 27 | ===========================================================
 28 | 
 29 | >>> pipeline3 = compose(
 30 | ...     "pipeline3",
 31 | ...     operation(name="op1", needs=["a", "b"], provides="aa")(lambda a, b: [a, b]),
 32 | ... )
 33 | 
 34 | .. graphtik::
 35 | 
 36 | 
 37 | 3. `graphtik` inherit from doctest-block with :graphvar:
 38 | ========================================================
 39 | 
 40 | >>> pipeline4 = compose(
 41 | ...     "pipeline4",
 42 | ...     operation(name="op1", needs=["a", "b"], provides="aa")(lambda a, b: [a, b]),
 43 | ... )
 44 | 
 45 | .. graphtik::
 46 |     :graphvar: pipeline4
 47 | 
 48 | 
 49 | 4. Image for :hide:
 50 | ===================
 51 | .. graphtik::
 52 |     :graphvar: pipeline1
 53 |     :hide: true
 54 |     :zoomable: false
 55 | 
 56 | 
 57 | 5. Nothing for :skipif:
 58 | =======================
 59 | .. graphtik::
 60 |     :graphvar: pipeline1
 61 |     :skipif: True
 62 | 
 63 | 
 64 | 
 65 | 6. Same name, different graph
 66 | =============================
 67 | .. graphtik::
 68 |     :zoomable:
 69 | 
 70 |     >>> pipeline5 = compose(
 71 |     ...     "pipeline1",
 72 |     ...     operation(name="op1", needs="a", provides="aa")(lambda a: a),
 73 |     ...     operation(name="op2", needs=["aa", "b"], provides="res")(lambda x, y: [x, y]),
 74 |     ... )
 75 | 
 76 | 
 77 | 7. Multiple plottables with prexistent
 78 | ======================================
 79 | Check order of doctest-globals even if item pre-exists:
 80 | 
 81 | .. graphtik::
 82 |     :zoomable-opts: {}
 83 | 
 84 |     >>> from graphtik import compose, operation
 85 |     >>> pipeline1 = compose(
 86 |     ...     "pipeline1",
 87 |     ...     operation(name="op1", needs=["a", "b"], provides="aa")(lambda a, b: [a, b])
 88 |     ... )
 89 |     >>> pipeline2 = compose(
 90 |     ...     "pipeline2",
 91 |     ...     operation(name="op1", needs=["a", "b"], provides="aa")(lambda a, b: [a, b])
 92 |     ... )
 93 | 
 94 | 
 95 | 8. Multiple plottables ignoring 1st
 96 | ===================================
 97 | .. graphtik::
 98 | 
 99 |     >>> from graphtik import compose, operation
100 |     >>> pipeline1 = compose(
101 |     ...     "pipelineA",
102 |     ...     operation(name="op1", needs=["A", "b"], provides="aa")(lambda a, b: [a, b])
103 |     ... )
104 | 
105 |     >>> pipeline2 = compose(
106 |     ...     "pipelineB",
107 |     ...     operation(name="op1", needs=["a", "B"], provides="aa")(lambda a, b: [a, b])
108 |     ... )
109 | 


--------------------------------------------------------------------------------
/test/sphinxext/test_directive.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Pytest-ing sphinx directives is `yet undocumented
  3 | <https://github.com/sphinx-doc/sphinx/issues/7008>`_
  4 | and as explained also in `this sourceforge <>`_ thread,
  5 | you may learn more from the `test-cases in the *sphinx* sources
  6 | <https://github.com/sphinx-doc/sphinx/blob/master/tests/test_ext_doctest.py>`_
  7 | or `similar projects
  8 | <https://github.com/pauleveritt/customizing_sphinx/blob/master/tests/integration/test_directive.py>`_.
  9 | """
 10 | import os.path as osp
 11 | import re
 12 | import xml.etree
 13 | from typing import Dict
 14 | from xml.etree.ElementTree import Element
 15 | 
 16 | import html5lib
 17 | import pytest
 18 | 
 19 | from graphtik.sphinxext import _image_formats
 20 | 
 21 | from ..helpers import attr_check, check_xpath, flat_dict
 22 | 
 23 | etree_cache: Dict[str, Element] = {}
 24 | 
 25 | 
 26 | @pytest.fixture(scope="module")
 27 | def cached_etree_parse() -> xml.etree:
 28 |     ## Adapted from sphinx testing
 29 |     def parse(fpath):
 30 |         cache_key = (fpath, fpath.stat().st_mtime)
 31 |         if cache_key in etree_cache:
 32 |             return etree_cache[fpath]
 33 |         try:
 34 |             with (fpath).open("rb") as fp:
 35 |                 etree: xml.etree = html5lib.HTMLParser(
 36 |                     namespaceHTMLElements=False
 37 |                 ).parse(fp)
 38 | 
 39 |             # etree_cache.clear() # WHY?
 40 |             etree_cache[cache_key] = etree
 41 |             return etree
 42 |         except Exception as ex:
 43 |             raise Exception(f"Parsing document {fpath} failed due to: {ex}") from ex
 44 | 
 45 |     yield parse
 46 |     etree_cache.clear()
 47 | 
 48 | 
 49 | @pytest.fixture(params=_image_formats)
 50 | def img_format(request):
 51 |     return request.param
 52 | 
 53 | 
 54 | @pytest.mark.sphinx(buildername="html", testroot="graphtik-directive")
 55 | @pytest.mark.test_params(shared_result="test_count_image_files")
 56 | def test_html(make_app, app_params, img_format, cached_etree_parse):
 57 |     fname = "index.html"
 58 |     args, kwargs = app_params
 59 |     app = make_app(
 60 |         *args,
 61 |         confoverrides={"graphtik_default_graph_format": img_format},
 62 |         freshenv=True,
 63 |         **kwargs,
 64 |     )
 65 |     fname = app.outdir / fname
 66 |     print(fname)
 67 | 
 68 |     ## Clean outdir from previous build to enact re-build.
 69 |     #
 70 |     try:
 71 |         app.outdir.rmtree(ignore_errors=True)
 72 |     except Exception:
 73 |         pass  # the 1st time should fail.
 74 |     finally:
 75 |         app.outdir.makedirs(exist_ok=True)
 76 | 
 77 |     app.build(True)
 78 | 
 79 |     image_dir = app.outdir / "_images"
 80 | 
 81 |     if img_format is None:
 82 |         img_format = "svg"
 83 | 
 84 |     image_files = image_dir.listdir()
 85 |     nimages = 9
 86 |     if img_format == "png":
 87 |         # x2 files for image-maps file.
 88 |         tag = "img"
 89 |         uri_attr = "src"
 90 |     else:
 91 |         tag = "object"
 92 |         uri_attr = "data"
 93 | 
 94 |     assert all(f.endswith(img_format) for f in image_files)
 95 |     assert len(image_files) == nimages - 3  # -1 skipIf=true, -1 same name, ???
 96 | 
 97 |     etree = cached_etree_parse(fname)
 98 |     check_xpath(
 99 |         etree,
100 |         fname,
101 |         f".//{tag}",
102 |         attr_check(
103 |             "alt",
104 |             "pipeline1",
105 |             r"'aa': \[1, 2\]",
106 |             "pipeline3",
107 |             "pipeline4",
108 |             "pipeline1",
109 |             "pipelineB",
110 |             count=True,
111 |         ),
112 |     )
113 |     check_xpath(
114 |         etree,
115 |         fname,
116 |         f".//{tag}",
117 |         attr_check(
118 |             uri_attr,
119 |         ),
120 |     )
121 |     check_xpath(etree, fname, ".//*[@class='caption-text']", "Solved ")
122 | 
123 | 
124 | def _count_nodes(count):
125 |     def checker(nodes):
126 |         assert len(nodes) == count
127 | 
128 |     return checker
129 | 
130 | 
131 | @pytest.mark.sphinx(buildername="html", testroot="graphtik-directive")
132 | @pytest.mark.test_params(shared_result="default_format")
133 | def test_zoomable_svg(app, cached_etree_parse):
134 |     app.build()
135 |     fname = "index.html"
136 |     print(app.outdir / fname)
137 | 
138 |     nimages = 9
139 |     etree = cached_etree_parse(app.outdir / fname)
140 |     check_xpath(
141 |         etree,
142 |         fname,
143 |         f".//object[@class='graphtik-zoomable-svg']",
144 |         _count_nodes(nimages - 3),  # -zoomable-options(BUG), -skipIf=true, -same-name
145 |     )
146 |     check_xpath(
147 |         etree,
148 |         fname,
149 |         f".//object[@data-graphtik-svg-zoom-options]",
150 |         _count_nodes(nimages - 3),  # see above
151 |     )
152 | 


--------------------------------------------------------------------------------
/test/sphinxext/test_image_purgatory.py:
--------------------------------------------------------------------------------
 1 | from pathlib import Path
 2 | from typing import List
 3 | 
 4 | import pytest
 5 | 
 6 | from graphtik.sphinxext import DocFilesPurgatory, _image_formats
 7 | 
 8 | 
 9 | @pytest.fixture
10 | def img_docs() -> List[str]:
11 |     return [f"d{i}" for i in range(3)]
12 | 
13 | 
14 | @pytest.fixture
15 | def img_files(tmpdir) -> List[Path]:
16 |     files = [tmpdir.join(f"f{i}") for i in range(3)]
17 |     for f in files:
18 |         f.ensure()
19 |     return [Path(i) for i in files]
20 | 
21 | 
22 | @pytest.fixture
23 | def img_reg(img_docs, img_files) -> DocFilesPurgatory:
24 |     img_reg = DocFilesPurgatory()
25 | 
26 |     img_reg.register_doc_fpath(img_docs[0], img_files[0])
27 |     img_reg.register_doc_fpath(img_docs[0], img_files[1])
28 |     img_reg.register_doc_fpath(img_docs[1], img_files[0])
29 |     img_reg.register_doc_fpath(img_docs[2], img_files[2])
30 | 
31 |     return img_reg
32 | 
33 | 
34 | def test_image_purgatory(img_docs, img_files, img_reg):
35 |     for _ in range(2):
36 |         img_reg.purge_doc(img_docs[2])
37 |         assert list(img_reg.doc_fpaths) == img_docs[:2]
38 |         assert img_files[0].exists()
39 |         assert img_files[1].exists()
40 |         assert not img_files[2].exists()
41 | 
42 |     for _ in range(2):
43 |         img_reg.purge_doc(img_docs[1])
44 |         assert list(img_reg.doc_fpaths) == img_docs[:1]
45 |         assert img_files[0].exists()
46 |         assert img_files[1].exists()
47 |         assert not img_files[2].exists()
48 | 
49 |     img_reg.purge_doc(img_docs[0])
50 |     assert not img_reg.doc_fpaths
51 |     assert not img_files[0].exists()
52 |     assert not img_files[1].exists()
53 |     assert not img_files[2].exists()
54 | 
55 |     img_reg.purge_doc(img_docs[0])
56 |     img_reg.purge_doc(img_docs[1])
57 |     img_reg.purge_doc(img_docs[2])
58 | 


--------------------------------------------------------------------------------
/test/test_combine.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2020-2020, Kostis Anagnostopoulos;
  2 | # Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms.
  3 | """Test renames, :term:`operation nesting` & :term:`operation merging`. """
  4 | import re
  5 | from functools import partial
  6 | from operator import add, mul, sub
  7 | from textwrap import dedent
  8 | 
  9 | import pytest
 10 | 
 11 | from graphtik import compose, operation, sfxed, token, vararg
 12 | from graphtik.fnop import Operation
 13 | from graphtik.modifier import dep_renamed
 14 | 
 15 | from .helpers import abspow, addall
 16 | 
 17 | 
 18 | def test_compose_rename_dict(caplog):
 19 |     pip = compose(
 20 |         "t",
 21 |         operation(str, "op1", provides=["a", "aa"]),
 22 |         operation(
 23 |             str,
 24 |             "op2",
 25 |             needs="a",
 26 |             provides=["b", token("c")],
 27 |             aliases=[("b", "B"), ("b", "p")],
 28 |         ),
 29 |         nest={"op1": "OP1", "op2": lambda n: "OP2", "a": "A", "b": "bb"},
 30 |     )
 31 |     print(str(pip))
 32 |     assert str(pip) == (
 33 |         "Pipeline('t', needs=['A'], "
 34 |         "provides=['A', 'aa', 'bb', 'c'($), 'B', 'p'], x2 ops: OP1, OP2)"
 35 |     )
 36 |     print(str(pip.ops))
 37 |     assert (
 38 |         str(pip.ops)
 39 |         == dedent(
 40 |             """
 41 |         [FnOp(name='OP1', provides=['A', 'aa'], fn='str'),
 42 |          FnOp(name='OP2', needs=['A'], provides=['bb', 'c'($), 'B', 'p'],
 43 |          aliases=[('bb', 'B'), ('bb', 'p')], fn='str')]
 44 |     """
 45 |         ).replace("\n", "")
 46 |     )
 47 | 
 48 | 
 49 | def test_compose_rename_dict_non_str(caplog):
 50 |     pip = compose(
 51 |         "t",
 52 |         operation(str, "op1"),
 53 |         operation(str, "op2"),
 54 |         nest={"op1": 1},
 55 |     )
 56 |     exp = "Pipeline('t', x2 ops: op1, op2)"
 57 |     print(pip)
 58 |     assert str(pip) == exp
 59 |     exp = "Pipeline('t', x2 ops: t.op1, op2)"
 60 |     pip = compose("t", pip, nest={"op1": 1, "op2": 0})
 61 |     assert str(pip) == exp
 62 |     pip = compose("t", pip, nest={"op1": 1, "op2": ""})
 63 |     assert str(pip) == exp
 64 |     for record in caplog.records:
 65 |         assert "Failed to nest-rename" not in record.message
 66 | 
 67 | 
 68 | def test_compose_rename_bad_screamy(caplog):
 69 |     def screamy_nester(ren_args):
 70 |         raise RuntimeError("Bluff")
 71 | 
 72 |     with pytest.raises(RuntimeError, match="Bluff"):
 73 |         compose(
 74 |             "test_nest_err",
 75 |             operation(str, "op1"),
 76 |             operation(str, "op2"),
 77 |             nest=screamy_nester,
 78 |         )
 79 |     for record in caplog.records:
 80 |         if record.levelname == "WARNING":
 81 |             assert "name='op1', parent=None)" in record.message
 82 | 
 83 | 
 84 | def test_compose_rename_preserve_ops(caplog):
 85 |     pip = compose(
 86 |         "t",
 87 |         operation(str, "op1"),
 88 |         operation(str, "op2"),
 89 |         nest=lambda na: f"aa.{na.name}",
 90 |     )
 91 |     assert str(pip) == "Pipeline('t', x2 ops: aa.op1, aa.op2)"
 92 | 
 93 | 
 94 | def test_compose_merge_ops():
 95 |     def ops_only(ren_args):
 96 |         return ren_args.typ == "op"
 97 | 
 98 |     sum_op1 = operation(name="sum_op1", needs=["a", "b"], provides="sum1")(add)
 99 |     sum_op2 = operation(name="sum_op2", needs=["a", "b"], provides="sum2")(add)
100 |     sum_op3 = operation(name="sum_op3", needs=["sum1", "c"], provides="sum3")(add)
101 |     net1 = compose("my network 1", sum_op1, sum_op2, sum_op3)
102 | 
103 |     exp = {"a": 1, "b": 2, "c": 4, "sum1": 3, "sum2": 3, "sum3": 7}
104 |     sol = net1(a=1, b=2, c=4)
105 |     assert sol == exp
106 | 
107 |     sum_op4 = operation(name="sum_op1", needs=["d", "e"], provides="a")(add)
108 |     sum_op5 = operation(name="sum_op2", needs=["a", "f"], provides="b")(add)
109 | 
110 |     net2 = compose("my network 2", sum_op4, sum_op5)
111 |     exp = {"a": 3, "b": 7, "d": 1, "e": 2, "f": 4}
112 |     sol = net2(**{"d": 1, "e": 2, "f": 4})
113 |     assert sol == exp
114 | 
115 |     net3 = compose("merged", net1, net2, nest=ops_only)
116 |     exp = {
117 |         "a": 3,
118 |         "b": 7,
119 |         "c": 5,
120 |         "d": 1,
121 |         "e": 2,
122 |         "f": 4,
123 |         "sum1": 10,
124 |         "sum2": 10,
125 |         "sum3": 15,
126 |     }
127 |     sol = net3(c=5, d=1, e=2, f=4)
128 |     assert sol == exp
129 | 
130 |     assert repr(net3).startswith(
131 |         "Pipeline('merged', needs=['a', 'b', 'sum1', 'c', 'd', 'e', 'f'], "
132 |         "provides=['sum1', 'sum2', 'sum3', 'a', 'b'], x5 ops"
133 |     )
134 | 
135 | 
136 | def test_network_combine():
137 |     sum_op1 = operation(
138 |         name="sum_op1", needs=[vararg("a"), vararg("b")], provides="sum1"
139 |     )(addall)
140 |     sum_op2 = operation(name="sum_op2", needs=[vararg("a"), "b"], provides="sum2")(
141 |         addall
142 |     )
143 |     sum_op3 = operation(name="sum_op3", needs=["sum1", "c"], provides="sum3")(add)
144 |     net1 = compose("my network 1", sum_op1, sum_op2, sum_op3)
145 |     exp = {"a": 1, "b": 2, "c": 4, "sum1": 3, "sum2": 3, "sum3": 7}
146 |     assert net1(a=1, b=2, c=4) == exp
147 |     assert repr(net1).startswith(
148 |         "Pipeline('my network 1', needs=['a'(?), 'b', 'sum1', 'c'], "
149 |         "provides=['sum1', 'sum2', 'sum3'], x3 ops"
150 |     )
151 | 
152 |     sum_op4 = operation(name="sum_op1", needs=[vararg("a"), "b"], provides="sum1")(
153 |         addall
154 |     )
155 |     sum_op5 = operation(name="sum_op4", needs=["sum1", "b"], provides="sum2")(add)
156 |     net2 = compose("my network 2", sum_op4, sum_op5)
157 |     exp = {"a": 1, "b": 2, "sum1": 3, "sum2": 5}
158 |     assert net2(**{"a": 1, "b": 2}) == exp
159 |     assert repr(net2).startswith(
160 |         "Pipeline('my network 2', needs=['a'(?), 'b', 'sum1'], provides=['sum1', 'sum2'], x2 ops"
161 |     )
162 | 
163 |     net3 = compose("merged", net1, net2)
164 |     exp = {"a": 1, "b": 2, "c": 4, "sum1": 3, "sum2": 5, "sum3": 7}
165 |     assert net3(a=1, b=2, c=4) == exp
166 | 
167 |     assert repr(net3).startswith(
168 |         "Pipeline('merged', needs=['a'(?), 'b', 'sum1', 'c'], provides=['sum1', 'sum2', 'sum3'], x4 ops"
169 |     )
170 | 
171 |     ## Reverse ops, change results and `needs` optionality.
172 |     #
173 |     net3 = compose("merged", net2, net1)
174 |     exp = {"a": 1, "b": 2, "c": 4, "sum1": 3, "sum2": 3, "sum3": 7}
175 |     assert net3(**{"a": 1, "b": 2, "c": 4}) == exp
176 | 
177 |     assert repr(net3).startswith(
178 |         "Pipeline('merged', needs=['a'(?), 'b', 'sum1', 'c'], provides=['sum1', 'sum2', 'sum3'], x4 ops"
179 |     )
180 | 
181 | 
182 | def test_network_nest_in_doctests():
183 |     days_count = 3
184 | 
185 |     weekday = compose(
186 |         "weekday",
187 |         operation(str, name="wake up", needs="backlog", provides="tasks"),
188 |         operation(str, name="sleep", needs="tasks", provides="todos"),
189 |     )
190 | 
191 |     weekday = compose(
192 |         "weekday",
193 |         operation(
194 |             lambda t: (t[:-1], t[-1:]),
195 |             name="work!",
196 |             needs="tasks",
197 |             provides=["tasks_done", "todos"],
198 |         ),
199 |         operation(str, name="sleep"),
200 |         weekday,
201 |     )
202 |     assert len(weekday.ops) == 3
203 | 
204 |     weekday = compose("weekday", weekday, excludes="sleep")
205 |     assert len(weekday.ops) == 2
206 | 
207 |     weekdays = [weekday.withset(name=f"day {i}") for i in range(days_count)]
208 |     week = compose("week", *weekdays, nest=True)
209 |     assert len(week.ops) == 6
210 | 
211 |     def nester(ren_args):
212 |         if ren_args.name not in ("backlog", "tasks_done", "todos"):
213 |             return True
214 | 
215 |     week = compose("week", *weekdays, nest=nester)
216 |     assert len(week.ops) == 6
217 |     sol = week.compute({"backlog": "a lot!"})
218 |     assert sol == {
219 |         "backlog": "a lot!",
220 |         "day 0.tasks": "a lot!",
221 |         "tasks_done": "a lot",
222 |         "todos": "!",
223 |         "day 1.tasks": "a lot!",
224 |         "day 2.tasks": "a lot!",
225 |     }
226 | 
227 | 
228 | def test_compose_nest_dict(caplog):
229 |     pipe = compose(
230 |         "t",
231 |         compose(
232 |             "p1",
233 |             operation(
234 |                 str,
235 |                 name="op1",
236 |                 needs=[token("a"), "aa"],
237 |                 provides=[sfxed("S1", "g"), sfxed("S2", "h")],
238 |             ),
239 |         ),
240 |         compose(
241 |             "p2",
242 |             operation(
243 |                 str,
244 |                 name="op2",
245 |                 needs=token("a"),
246 |                 provides=["a", token("b")],
247 |                 aliases=[("a", "b")],
248 |             ),
249 |         ),
250 |         nest={
251 |             "op1": True,
252 |             "op2": lambda n: "p2.op2",
253 |             "aa": False,
254 |             token("a"): True,
255 |             "b": lambda n: f"PP.{n}",
256 |             sfxed("S1", "g"): True,
257 |             sfxed("S2", "h"): lambda n: dep_renamed(n, "ss2"),
258 |             token("b"): True,
259 |         },
260 |     )
261 |     got = str(pipe.ops)
262 |     print(got)
263 |     assert got == re.sub(
264 |         r"[\n ]{2,}",  # collapse all space-chars into a single space
265 |         " ",
266 |         """
267 |         [FnOp(name='p1.op1', needs=['p1.a'($), 'aa'],
268 |          provides=[sfxed('p1.S1', 'g'), sfxed('ss2', 'h')], fn='str'),
269 |         FnOp(name='p2.op2', needs=['p2.a'($)],
270 |          provides=['a', 'p2.b'($), 'PP.b'], aliases=[('a', 'PP.b')], fn='str')]
271 | 
272 |         """.strip(),
273 |     )
274 |     for record in caplog.records:
275 |         assert record.levelname != "WARNING"
276 | 
277 | 
278 | @pytest.mark.parametrize("bools", range(4))
279 | def test_combine_networks(exemethod, bools):
280 |     # Code from `compose.rst` examples
281 |     if not exemethod:
282 |         return
283 | 
284 |     parallel1 = bools >> 0 & 1
285 |     parallel2 = bools >> 1 & 1
286 | 
287 |     graphop = compose(
288 |         "graphop",
289 |         operation(name="mul1", needs=["a", "b"], provides=["ab"])(mul),
290 |         operation(name="sub1", needs=["a", "ab"], provides=["a-ab"])(sub),
291 |         operation(name="abspow1", needs=["a-ab"], provides=["|a-ab|³"])(
292 |             partial(abspow, p=3)
293 |         ),
294 |         parallel=parallel1,
295 |     )
296 | 
297 |     assert graphop.compute({"a-ab": -8}) == {"a-ab": -8, "|a-ab|³": 512}
298 | 
299 |     bigger_graph = compose(
300 |         "bigger_graph",
301 |         graphop,
302 |         operation(name="sub2", needs=["a-ab", "c"], provides="a-ab_minus_c")(sub),
303 |         parallel=parallel2,
304 |         nest=lambda ren_args: ren_args.typ == "op",
305 |     )
306 |     ## Ensure all old-nodes were prefixed.
307 |     #
308 |     old_nodes = graphop.net.graph.nodes
309 |     new_nodes = bigger_graph.net.graph.nodes
310 |     for old_node in old_nodes:
311 |         if isinstance(old_node, Operation):
312 |             assert old_node not in new_nodes
313 |         else:
314 |             assert old_node in new_nodes
315 | 
316 |     sol = bigger_graph.compute({"a": 2, "b": 5, "c": 5}, ["a-ab_minus_c"])
317 |     assert sol == {"a-ab_minus_c": -13}
318 | 
319 |     ## Test Plots
320 | 
321 |     ## Ensure all old-nodes were prefixed.
322 |     #
323 |     # Access all nodes from Network, where no "after pruning" cluster exists.
324 |     old_nodes = [n for n in graphop.net.plot().get_nodes()]
325 |     new_node_names = [n.get_name() for n in bigger_graph.net.plot().get_nodes()]
326 | 
327 |     for old_node in old_nodes:
328 |         if old_node.get_shape() == "plain":  # Operation
329 |             assert old_node.get_name() not in new_node_names
330 |         else:
331 |             # legend-node included here.`
332 |             assert old_node.get_name() in new_node_names
333 | 


--------------------------------------------------------------------------------
/test/test_execution.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2020-2020, Kostis Anagnostopoulos;
  2 | # Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms.
  3 | """Test :term:`parallel`, :term:`marshalling` and other :term:`execution` related stuff. """
  4 | import io
  5 | import os
  6 | from functools import partial
  7 | from multiprocessing import cpu_count
  8 | from multiprocessing import dummy as mp_dummy
  9 | from operator import mul, sub
 10 | from textwrap import dedent
 11 | from time import sleep, time
 12 | 
 13 | import pandas as pd
 14 | import pytest
 15 | from pandas.testing import assert_frame_equal
 16 | 
 17 | from graphtik import AbortedException, compose, hcat, modify, operation, optional, vcat
 18 | from graphtik.config import abort_run, execution_pool_plugged
 19 | from graphtik.execution import OpTask, task_context
 20 | 
 21 | from .helpers import abspow, dummy_sol, exe_params
 22 | 
 23 | 
 24 | @pytest.mark.xfail(reason="Spurious passes when threading with on low-cores?")
 25 | def test_task_context(exemethod, request):
 26 |     def check_task_context():
 27 |         sleep(0.15)
 28 |         assert task_context.get().op == next(iop), "Corrupted task-context"
 29 | 
 30 |     n_ops = 10
 31 |     pipe = compose(
 32 |         "t",
 33 |         *(
 34 |             operation(check_task_context, f"op{i}", provides=f"{i}")
 35 |             for i in range(n_ops)
 36 |         ),
 37 |         parallel=exemethod,
 38 |     )
 39 |     iop = iter(pipe.ops)
 40 | 
 41 |     print(exe_params, cpu_count())
 42 |     err = None
 43 |     if exe_params.proc and exe_params.marshal:
 44 |         err = Exception("^Error sending result")
 45 |     elif exe_params.parallel and exe_params.marshal:
 46 |         err = AssertionError("^Corrupted task-context")
 47 |     elif exe_params.parallel and not os.environ.get("TRAVIS"):
 48 |         # Travis has low parallelism and error does not surface
 49 |         err = AssertionError("^Corrupted task-context")
 50 | 
 51 |     if err:
 52 |         with pytest.raises(type(err), match=str(err)):
 53 |             pipe.compute()
 54 |         raise pytest.xfail("Cannot marshal parallel processes with `task_context` :-(.")
 55 |     else:
 56 |         pipe.compute()
 57 |         with pytest.raises(StopIteration):
 58 |             next(iop)
 59 | 
 60 | 
 61 | @pytest.mark.xfail(
 62 |     reason="Spurious copied-reversed graphs in Travis, with dubious cause...."
 63 | )
 64 | def test_multithreading_plan_execution():
 65 |     # Compose the mul, sub, and abspow operations into a computation graph.
 66 |     # From Huygn's test-code given in yahoo/graphkit#31
 67 |     graph = compose(
 68 |         "graph",
 69 |         operation(name="mul1", needs=["a", "b"], provides=["ab"])(mul),
 70 |         operation(name="sub1", needs=["a", "ab"], provides=["a-ab"])(sub),
 71 |         operation(name="abspow1", needs=["a-ab"], provides=["|a-ab|³"])(
 72 |             partial(abspow, p=3)
 73 |         ),
 74 |     )
 75 | 
 76 |     with mp_dummy.Pool(int(2 * cpu_count())) as pool, execution_pool_plugged(pool):
 77 |         pool.map(
 78 |             # lambda i: graph.withset(name='graph').compute(
 79 |             lambda i: graph.compute({"a": 2, "b": 5}, ["a-ab", "|a-ab|³"]),
 80 |             range(300),
 81 |         )
 82 | 
 83 | 
 84 | @pytest.mark.slow
 85 | def test_parallel_execution(exemethod):
 86 |     if not exemethod:
 87 |         return
 88 | 
 89 |     delay = 0.5
 90 | 
 91 |     def fn(x):
 92 |         sleep(delay)
 93 |         print("fn %s" % (time() - t0))
 94 |         return 1 + x
 95 | 
 96 |     def fn2(a, b):
 97 |         sleep(delay)
 98 |         print("fn2 %s" % (time() - t0))
 99 |         return a + b
100 | 
101 |     def fn3(z, k=1):
102 |         sleep(delay)
103 |         print("fn3 %s" % (time() - t0))
104 |         return z + k
105 | 
106 |     pipeline = compose(
107 |         "l",
108 |         # the following should execute in parallel under threaded execution mode
109 |         operation(name="a", needs="x", provides="ao")(fn),
110 |         operation(name="b", needs="x", provides="bo")(fn),
111 |         # this should execute after a and b have finished
112 |         operation(name="c", needs=["ao", "bo"], provides="co")(fn2),
113 |         operation(name="d", needs=["ao", optional("k")], provides="do")(fn3),
114 |         operation(name="e", needs=["ao", "bo"], provides="eo")(fn2),
115 |         operation(name="f", needs="eo", provides="fo")(fn),
116 |         operation(name="g", needs="fo", provides="go")(fn),
117 |         nest=False,
118 |     )
119 | 
120 |     t0 = time()
121 |     result_threaded = pipeline.withset(parallel=True).compute(
122 |         {"x": 10}, ["co", "go", "do"]
123 |     )
124 |     # print("threaded result")
125 |     # print(result_threaded)
126 | 
127 |     t0 = time()
128 |     pipeline = pipeline.withset(parallel=False)
129 |     result_sequential = pipeline.compute({"x": 10}, ["co", "go", "do"])
130 |     # print("sequential result")
131 |     # print(result_sequential)
132 | 
133 |     # make sure results are the same using either method
134 |     assert result_sequential == result_threaded
135 | 
136 | 
137 | @pytest.mark.slow
138 | @pytest.mark.xfail(
139 |     reason="Spurious copied-reversed graphs in Travis, with dubious cause...."
140 | )
141 | def test_multi_threading_computes():
142 |     import random
143 | 
144 |     def op_a(a, b):
145 |         sleep(random.random() * 0.02)
146 |         return a + b
147 | 
148 |     def op_b(c, b):
149 |         sleep(random.random() * 0.02)
150 |         return c + b
151 | 
152 |     def op_c(a, b):
153 |         sleep(random.random() * 0.02)
154 |         return a * b
155 | 
156 |     pipeline = compose(
157 |         "pipeline",
158 |         operation(name="op_a", needs=["a", "b"], provides="c")(op_a),
159 |         operation(name="op_b", needs=["c", "b"], provides="d")(op_b),
160 |         operation(name="op_c", needs=["a", "b"], provides="e")(op_c),
161 |         nest=False,
162 |     )
163 | 
164 |     def infer(i):
165 |         # data = open("616039-bradpitt.jpg").read()
166 |         outputs = ["c", "d", "e"]
167 |         results = pipeline.compute({"a": 1, "b": 2}, outputs)
168 |         assert tuple(sorted(results.keys())) == tuple(sorted(outputs)), (
169 |             outputs,
170 |             results,
171 |         )
172 |         return results
173 | 
174 |     N = 33
175 |     for i in range(13, 61):
176 |         with mp_dummy.Pool(i) as pool:
177 |             pool.map(infer, range(N))
178 | 
179 | 
180 | def test_abort(exemethod):
181 |     pipeline = compose(
182 |         "pipeline",
183 |         operation(fn=None, name="A", needs=["a"], provides=["b"]),
184 |         operation(name="B", needs=["b"], provides=["c"])(lambda x: abort_run()),
185 |         operation(fn=None, name="C", needs=["c"], provides=["d"]),
186 |         parallel=exemethod,
187 |     )
188 |     with pytest.raises(AbortedException) as exinfo:
189 |         pipeline(a=1)
190 | 
191 |     exp = {"a": 1, "b": 1, "c": None}
192 |     solution = exinfo.value.args[0]
193 |     assert solution == exp
194 |     assert exinfo.value.jetsam["solution"] == exp
195 |     assert solution.executed == {"A": {"b": 1}, "B": {"c": None}}
196 | 
197 |     pipeline = compose(
198 |         "pipeline",
199 |         operation(fn=None, name="A", needs=["a"], provides=["b"]),
200 |         parallel=exemethod,
201 |     )
202 |     assert pipeline.compute({"a": 1}) == {"a": 1, "b": 1}
203 | 
204 | 
205 | def test_solution_copy(samplenet):
206 |     sol = samplenet(a=1, b=2)
207 |     assert sol == sol.copy()
208 | 
209 | 
210 | def test_solution_df_concat_delay_groups(monkeypatch):
211 |     concat_args = []
212 | 
213 |     def my_concat(dfs, *args, **kwds):
214 |         concat_args.append(dfs)
215 |         return orig_concat(dfs, *args, **kwds)
216 | 
217 |     orig_concat = pd.concat
218 |     monkeypatch.setattr(pd, "concat", my_concat)
219 | 
220 |     df = pd.DataFrame({"doc": [1, 2]})
221 |     axis_names = ["l1"]
222 |     df.index.names = df.columns.names = axis_names
223 |     # for when debugging
224 |     _orig_doc = {"a": df}
225 |     sol = dummy_sol(_orig_doc.copy())
226 | 
227 |     val1 = pd.Series([3, 4], name="val1")
228 |     val11 = val1.copy()
229 |     val11.name = "val11"
230 |     val111 = val1.copy()
231 |     val111.name = "val111"
232 |     val2 = (
233 |         pd.Series([11, 22, 33, 44], index=["doc", "val1", "val11", "val111"])
234 |         .to_frame()
235 |         .T
236 |     )
237 |     val3 = pd.Series([5, 6, 7, 8], name="val3")
238 |     val4 = pd.Series([44, 55, 66], index=["doc", "val1", "val3"]).to_frame().T
239 | 
240 |     path_values = [
241 |         (hcat("a/H1"), val1),
242 |         (hcat("a/H"), val11),
243 |         (hcat("a/H3"), val111),
244 |         (vcat("a/V2"), val2),
245 |         (vcat("a/V"), val2),
246 |         (hcat("a/H"), val3),
247 |         (vcat("a/H"), val4),
248 |         (modify("a/INT"), 0),
249 |         (vcat("a/H"), val4),
250 |     ]
251 |     sol.update(path_values)
252 | 
253 |     df = sol["a"]
254 |     exp_csv = """
255 |         l1,doc,val11,val3,val1,val111,INT
256 |         0,1.0,3.0,5.0,3.0,3.0,0
257 |         1,2.0,4.0,6.0,4.0,4.0,0
258 |         2,,,7.0,,,0
259 |         3,,,8.0,,,0
260 |         0,44.0,,66.0,55.0,,0
261 |         0,44.0,,66.0,55.0,,0
262 |         0,11.0,33.0,,22.0,44.0,0
263 |         0,11.0,33.0,,22.0,44.0,0
264 |         """
265 | 
266 |     exp = pd.read_csv(io.StringIO(dedent(exp_csv)), index_col=0)
267 | 
268 |     print(df.to_csv())
269 |     assert [len(i) for i in concat_args] == [5, 5]
270 |     assert_frame_equal(df, exp)
271 | 


--------------------------------------------------------------------------------
/test/test_planning.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2020-2020, Kostis Anagnostopoulos;
  2 | # Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms.
  3 | """mostly :mod:`networkx` routines tests"""
  4 | 
  5 | import networkx as nx
  6 | import pytest
  7 | from networkx.readwrite.edgelist import parse_edgelist
  8 | 
  9 | from graphtik import operation
 10 | from graphtik.planning import (
 11 |     Network,
 12 |     yield_also_chaindocs,
 13 |     yield_also_subdocs,
 14 |     yield_also_superdocs,
 15 |     yield_chaindocs,
 16 |     yield_subdocs,
 17 |     yield_superdocs,
 18 | )
 19 | 
 20 | 
 21 | @pytest.fixture
 22 | def g():
 23 |     g = parse_edgelist(
 24 |         """
 25 |         root    d1                      1
 26 |                 d1      d11             1
 27 |                 d1      d12             1
 28 |         root    d2                      1
 29 |                 d2      d21             1
 30 |                         d21     d211    1
 31 | 
 32 |         # Irrelevant nodes
 33 |         root    foo
 34 |                 d1      bar
 35 |                         d11     baz
 36 |     """.splitlines(),
 37 |         create_using=nx.DiGraph,
 38 |         data=[("subdoc", bool)],
 39 |     )
 40 |     return g
 41 | 
 42 | 
 43 | @pytest.fixture(
 44 |     params=[
 45 |         yield_also_subdocs,
 46 |         yield_also_superdocs,
 47 |         yield_also_chaindocs,
 48 |         lambda g, n, *args: yield_subdocs(g, [n], *args),
 49 |         lambda g, n, *args: yield_superdocs(g, [n], *args),
 50 |         lambda g, n, *args: yield_chaindocs(g, [n], *args),
 51 |     ]
 52 | )
 53 | def chain_fn(request):
 54 |     return request.param
 55 | 
 56 | 
 57 | @pytest.mark.parametrize("bad", ["BAD", "/root/BAD", "root/d1/BAD" "root/d1/d11/BAD"])
 58 | def test_yield_chained_docs_unknown(g, chain_fn, bad):
 59 |     assert list(chain_fn(g, bad)) == []
 60 |     assert list(chain_fn(g, bad, ())) == []
 61 | 
 62 | 
 63 | @pytest.mark.parametrize("node", "foo bar baz".split())
 64 | def test_yield_chained_docs_regular(g, chain_fn, node):
 65 |     assert list(chain_fn(g, node)) == [node]
 66 |     assert list(chain_fn(g, node, ())) == [node]
 67 |     assert list(chain_fn(g, node, (node,))) == []
 68 | 
 69 | 
 70 | def test_yield_chained_docs_leaf(g):
 71 |     ## leaf-doc
 72 |     #
 73 |     assert list(yield_also_subdocs(g, "d11")) == ["d11"]
 74 |     assert list(yield_also_subdocs(g, "d11", ())) == ["d11"]
 75 |     assert list(yield_also_superdocs(g, "d11")) == ["d11", "d1", "root"]
 76 |     assert list(yield_also_superdocs(g, "d11", ())) == ["d11", "d1", "root"]
 77 |     assert list(yield_also_chaindocs(g, "d11")) == ["d11", "d1", "root"]
 78 |     assert list(yield_also_chaindocs(g, "d11", ())) == ["d11", "d1", "root"]
 79 | 
 80 |     assert list(yield_subdocs(g, ["d11"])) == ["d11"]
 81 |     assert list(yield_subdocs(g, ["d11"], ())) == ["d11"]
 82 |     assert list(yield_superdocs(g, ["d11"])) == ["d11", "d1", "root"]
 83 |     assert list(yield_superdocs(g, ["d11"], ())) == ["d11", "d1", "root"]
 84 |     assert list(yield_chaindocs(g, ["d11"])) == ["d11", "d1", "root"]
 85 |     assert list(yield_chaindocs(g, ["d11"], ())) == ["d11", "d1", "root"]
 86 | 
 87 | 
 88 | def test_yield_chained_docs_inner(g):
 89 |     ## inner-node
 90 |     #
 91 |     assert list(yield_also_subdocs(g, "d1")) == ["d1", "d11", "d12"]
 92 |     assert list(yield_also_subdocs(g, "d1", ())) == ["d1", "d11", "d12"]
 93 |     assert list(yield_also_superdocs(g, "d1")) == ["d1", "root"]
 94 |     assert list(yield_also_superdocs(g, "d1", ())) == ["d1", "root"]
 95 |     assert list(yield_also_chaindocs(g, "d1")) == ["d1", "d11", "d12", "root"]
 96 |     assert list(yield_also_chaindocs(g, "d1", ())) == ["d1", "d11", "d12", "root"]
 97 | 
 98 |     assert list(yield_subdocs(g, ["d1"])) == ["d1", "d11", "d12"]
 99 |     assert list(yield_subdocs(g, ["d1"], ())) == ["d1", "d11", "d12"]
100 |     assert list(yield_superdocs(g, ["d1"])) == ["d1", "root"]
101 |     assert list(yield_superdocs(g, ["d1"], ())) == ["d1", "root"]
102 |     assert list(yield_chaindocs(g, ["d1"])) == ["d1", "d11", "d12", "root"]
103 |     assert list(yield_chaindocs(g, ["d1"], ())) == ["d1", "d11", "d12", "root"]
104 | 
105 | 
106 | def test_yield_chained_docs_root(g):
107 |     ## root-doc
108 |     #
109 |     assert list(yield_also_subdocs(g, "root")) == [
110 |         "root",
111 |         "d1",
112 |         "d11",
113 |         "d12",
114 |         "d2",
115 |         "d21",
116 |         "d211",
117 |     ]
118 |     assert list(yield_also_subdocs(g, "root", ())) == [
119 |         "root",
120 |         "d1",
121 |         "d11",
122 |         "d12",
123 |         "d2",
124 |         "d21",
125 |         "d211",
126 |     ]
127 |     assert list(yield_also_superdocs(g, "root")) == ["root"]
128 |     assert list(yield_also_superdocs(g, "root", ())) == ["root"]
129 |     assert list(yield_also_chaindocs(g, "root")) == [
130 |         "root",
131 |         "d1",
132 |         "d11",
133 |         "d12",
134 |         "d2",
135 |         "d21",
136 |         "d211",
137 |     ]
138 |     assert list(yield_also_chaindocs(g, "root", ())) == [
139 |         "root",
140 |         "d1",
141 |         "d11",
142 |         "d12",
143 |         "d2",
144 |         "d21",
145 |         "d211",
146 |     ]
147 | 
148 | 
149 | @pytest.mark.parametrize(
150 |     "ops, err",
151 |     [
152 |         (
153 |             lambda: [operation(str, "BOOM", provides="BOOM")],
154 |             r"Name of provides\('BOOM'\) clashed with a same-named operation",
155 |         ),
156 |         (
157 |             lambda: [operation(str, "BOOM", needs="BOOM")],
158 |             r"Name of operation\(BOOM\) clashed with a same-named dependency",
159 |         ),
160 |         (
161 |             lambda: [operation(str, "BOOM", provides="a", aliases={"a": "BOOM"})],
162 |             r"Name of provides\('BOOM'\) clashed with a same-named operation",
163 |         ),
164 |         (
165 |             lambda: [operation(str, "op1", provides="BOOM"), operation(str, "BOOM")],
166 |             r"Name of operation\(BOOM\) clashed with a same-named dependency",
167 |         ),
168 |         ## x2 ops
169 |         (
170 |             lambda: [operation(str, "BOOM"), operation(str, "op2", "BOOM")],
171 |             r"Name of needs\('BOOM'\) clashed with a same-named operation",
172 |         ),
173 |         (
174 |             lambda: [operation(str, "op1", needs="BOOM"), operation(str, "BOOM")],
175 |             r"Name of operation\(BOOM\) clashed with a same-named dependency",
176 |         ),
177 |         (
178 |             lambda: [
179 |                 operation(str, "op1", provides="a", aliases={"a": "BOOM"}),
180 |                 operation(str, "BOOM"),
181 |             ],
182 |             r"Name of operation\(BOOM\) clashed with a same-named dependency",
183 |         ),
184 |     ],
185 | )
186 | def test_node_clashes(ops, err):
187 |     with pytest.raises(ValueError, match=err):
188 |         Network(*ops())
189 | 


--------------------------------------------------------------------------------
/test/test_remerge.py:
--------------------------------------------------------------------------------
  1 | # coding: utf-8
  2 | 
  3 | # Standard Library
  4 | from pprint import pprint
  5 | 
  6 | import pytest
  7 | 
  8 | # Gitlab Project Configurator Modules
  9 | # from gpc.helpers.remerge import remerge
 10 | from graphtik.plot import remerge
 11 | 
 12 | 
 13 | def test_override_string():
 14 |     defaults = {"key_to_override": "value_from_defaults"}
 15 | 
 16 |     first_override = {"key_to_override": "value_from_first_override"}
 17 | 
 18 |     source_map = {}
 19 |     merged = remerge(
 20 |         ("defaults", defaults),
 21 |         ("first_override", first_override),
 22 |         source_map=source_map,
 23 |     )
 24 | 
 25 |     expected_merged = {"key_to_override": "value_from_first_override"}
 26 |     assert merged == expected_merged
 27 |     assert source_map == {("key_to_override",): "first_override"}
 28 | 
 29 |     merged = remerge(defaults, first_override, source_map=None)
 30 |     assert merged == expected_merged
 31 | 
 32 | 
 33 | def test_override_subdict():
 34 |     defaults = {
 35 |         "subdict": {
 36 |             "other_subdict": {
 37 |                 "key_to_override": "value_from_defaults",
 38 |                 "integer_to_override": 2222,
 39 |             }
 40 |         }
 41 |     }
 42 | 
 43 |     first_override = {
 44 |         "subdict": {
 45 |             "other_subdict": {
 46 |                 "key_to_override": "value_from_first_override",
 47 |                 "integer_to_override": 5555,
 48 |             }
 49 |         }
 50 |     }
 51 | 
 52 |     expected_merge = {
 53 |         "subdict": {
 54 |             "other_subdict": {
 55 |                 "integer_to_override": 5555,
 56 |                 "key_to_override": "value_from_first_override",
 57 |             }
 58 |         }
 59 |     }
 60 | 
 61 |     source_map = {}
 62 |     merged = remerge(
 63 |         ("defaults", defaults),
 64 |         ("first_override", first_override),
 65 |         source_map=source_map,
 66 |     )
 67 |     assert merged == expected_merge
 68 |     assert source_map == {
 69 |         ("subdict",): "first_override",
 70 |         ("subdict", "other_subdict"): "first_override",
 71 |         ("subdict", "other_subdict", "integer_to_override"): "first_override",
 72 |         ("subdict", "other_subdict", "key_to_override"): "first_override",
 73 |     }
 74 | 
 75 |     merged = remerge(defaults, first_override, source_map=None)
 76 |     assert merged == expected_merge
 77 | 
 78 | 
 79 | def test_override_list_append():
 80 |     defaults = {"list_to_append": [{"a": 1}]}
 81 |     first_override = {"list_to_append": [{"b": 1}]}
 82 | 
 83 |     source_map = {}
 84 |     merged = remerge(
 85 |         ("defaults", defaults),
 86 |         ("first_override", first_override),
 87 |         source_map=source_map,
 88 |     )
 89 |     expected_merged = {"list_to_append": [{"a": 1}, {"b": 1}]}
 90 | 
 91 |     assert merged == expected_merged
 92 |     assert source_map == {("list_to_append",): ["defaults", "first_override"]}
 93 | 
 94 |     merged = remerge(defaults, first_override, source_map=None)
 95 |     assert merged == expected_merged
 96 | 
 97 | 
 98 | def test_complex_dict():
 99 |     defaults = {
100 |         "key_to_override": "value_from_defaults",
101 |         "integer_to_override": 1111,
102 |         "list_to_append": [{"a": 1}],
103 |         "subdict": {
104 |             "other_subdict": {
105 |                 "key_to_override": "value_from_defaults",
106 |                 "integer_to_override": 2222,
107 |             },
108 |             "second_subdict": {
109 |                 "key_to_override": "value_from_defaults",
110 |                 "integer_to_override": 3333,
111 |             },
112 |         },
113 |     }
114 | 
115 |     first_override = {
116 |         "key_to_override": "value_from_first_override",
117 |         "integer_to_override": 4444,
118 |         "list_to_append": [{"b": 2}],
119 |         "subdict": {
120 |             "other_subdict": {
121 |                 "key_to_override": "value_from_first_override",
122 |                 "integer_to_override": 5555,
123 |             }
124 |         },
125 |         "added_in_first_override": "some_string",
126 |     }
127 | 
128 |     second_override = {
129 |         "subdict": {"second_subdict": {"key_to_override": "value_from_second_override"}}
130 |     }
131 | 
132 |     source_map = {}
133 |     merged = remerge(
134 |         ("defaults", defaults),
135 |         ("first_override", first_override),
136 |         ("second_override", second_override),
137 |         source_map=source_map,
138 |     )
139 |     print("")
140 |     print("'merged' dictionary:")
141 |     pprint(merged)
142 |     print("")
143 |     pprint(source_map)
144 |     print(len(source_map), "paths")
145 | 
146 |     assert merged["key_to_override"] == "value_from_first_override"
147 |     assert merged["integer_to_override"] == 4444
148 |     assert (
149 |         merged["subdict"]["other_subdict"]["key_to_override"]
150 |         == "value_from_first_override"
151 |     )
152 |     assert merged["subdict"]["other_subdict"]["integer_to_override"] == 5555
153 |     assert (
154 |         merged["subdict"]["second_subdict"]["key_to_override"]
155 |         == "value_from_second_override"
156 |     )
157 |     assert merged["subdict"]["second_subdict"]["integer_to_override"] == 3333
158 |     assert merged["added_in_first_override"] == "some_string"
159 |     assert merged["list_to_append"] == [{"a": 1}, {"b": 2}]
160 | 
161 | 
162 | _xfail = pytest.mark.xfail(reason="Brittle remerge()...")
163 | 
164 | 
165 | @pytest.mark.parametrize(
166 |     "inp, reverse, exp",
167 |     [
168 |         pytest.param(({1: None}, {}), 0, {}, marks=_xfail),
169 |         (({1: None}, {1: {}}), 0, {1: {}}),
170 |         (({1: None}, {1: []}), 0, {1: []}),
171 |         (({1: None}, {1: 33}), 0, {1: 33}),
172 |         pytest.param(({1: None}, {}), 1, {}, marks=_xfail),
173 |         pytest.param(({1: None}, {1: {}}), 1, {1: {}}, marks=_xfail),
174 |         pytest.param(({1: None}, {1: []}), 1, {1: []}, marks=_xfail),
175 |         pytest.param(({1: None}, {1: 33}), 1, {1: 33}, marks=_xfail),
176 |     ],
177 | )
178 | def test_none_values(inp, reverse, exp):
179 |     if reverse:
180 |         inp = reversed(inp)
181 |     assert remerge(*inp) == exp
182 | 
183 |     ## Test one level deeper
184 |     #
185 |     inp = tuple({"a": i} for i in inp)
186 |     exp = {"a": exp}
187 |     assert remerge(*inp) == exp
188 | 
189 | 
190 | @pytest.mark.parametrize(
191 |     "inp",
192 |     [
193 |         ## <don't collapse>
194 |         ({1: []}, {1: {}}),
195 |         ({1: {}}, {1: []}),
196 |         pytest.param(({1: []}, {1: "a"}), marks=_xfail),
197 |         pytest.param(({1: {}}, {1: "a"}), marks=_xfail),
198 |     ],
199 | )
200 | def test_incompatible_containers(inp):
201 |     with pytest.raises(TypeError, match="Incompatible types"):
202 |         remerge(*inp)
203 |     with pytest.raises(TypeError, match="Incompatible types"):
204 |         remerge(*reversed(inp))
205 | 
206 |     ## Test one level deeper
207 |     #
208 |     inp = tuple({"A": i} for i in inp)
209 |     with pytest.raises(TypeError, match="Incompatible types"):
210 |         remerge(*inp)
211 |     with pytest.raises(TypeError, match="Incompatible types"):
212 |         remerge(*reversed(inp))
213 | 


--------------------------------------------------------------------------------
/test/test_site.py:
--------------------------------------------------------------------------------
 1 | # Copyright 2016, Yahoo Inc.
 2 | # Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms.
 3 | import importlib
 4 | import io
 5 | import logging
 6 | import os
 7 | import os.path as osp
 8 | import re
 9 | import subprocess
10 | import sys
11 | 
12 | from docutils import core as dcore
13 | from readme_renderer import rst
14 | 
15 | proj_path = osp.join(osp.dirname(__file__), "..")
16 | 
17 | ########################
18 | ## Copied from Twine
19 | 
20 | # Regular expression used to capture and reformat docutils warnings into
21 | # something that a human can understand. This is loosely borrowed from
22 | # Sphinx: https://github.com/sphinx-doc/sphinx/blob
23 | # /c35eb6fade7a3b4a6de4183d1dd4196f04a5edaf/sphinx/util/docutils.py#L199
24 | _REPORT_RE = re.compile(
25 |     r"^<string>:(?P<line>(?:\d+)?): "
26 |     r"\((?P<level>DEBUG|INFO|WARNING|ERROR|SEVERE)/(\d+)?\) "
27 |     r"(?P<message>.*)",
28 |     re.DOTALL | re.MULTILINE,
29 | )
30 | 
31 | 
32 | class _WarningStream:
33 |     def __init__(self):
34 |         self.output = io.StringIO()
35 | 
36 |     def write(self, text):
37 |         matched = _REPORT_RE.search(text)
38 | 
39 |         if not matched:
40 |             self.output.write(text)
41 |             return
42 | 
43 |         self.output.write(
44 |             "line {line}: {level_text}: {message}\n".format(
45 |                 level_text=matched.group("level").capitalize(),
46 |                 line=matched.group("line"),
47 |                 message=matched.group("message").rstrip("\r\n"),
48 |             )
49 |         )
50 | 
51 |     def __repr__(self):
52 |         return self.output.getvalue()
53 | 
54 | 
55 | def test_README_as_PyPi_landing_page(monkeypatch):
56 |     long_desc = subprocess.check_output(
57 |         "python setup.py --long-description".split(), cwd=proj_path
58 |     )
59 |     assert long_desc
60 | 
61 |     err_stream = _WarningStream()
62 |     result = rst.render(
63 |         long_desc,
64 |         # The specific options are a selective copy of:
65 |         # https://github.com/pypa/readme_renderer/blob/master/readme_renderer/rst.py
66 |         stream=err_stream,
67 |         halt_level=2,  # 2=WARN, 1=INFO
68 |     )
69 |     assert result, err_stream
70 | 
71 | 
72 | def test_sphinx_html(monkeypatch):
73 |     from sphinx.cmd import build
74 | 
75 |     # Fail on warnings, but don't rebuild all files (no `-a`),
76 |     # rm -r ./build/sphinx/ # to fully test site.
77 |     build_options = [
78 |         "-W",
79 |         # "-E",
80 |         "-D",
81 |         "graphtik_warning_is_error=true",
82 |         "--color",
83 |         "-j",
84 |         "auto",
85 |     ]
86 |     if logging.getLogger(__name__).isEnabledFor(logging.INFO):
87 |         build_options.append("-v")
88 |     site_args = [*build_options, "docs/source/", "docs/build"]
89 |     monkeypatch.chdir(proj_path)
90 |     ret = build.build_main(site_args)
91 |     assert not ret
92 | 


--------------------------------------------------------------------------------